From 259be14fad04e08b8f8219fd854b0f5c7a888b54 Mon Sep 17 00:00:00 2001 From: Valter Silva Date: Thu, 15 Oct 2026 13:50:50 +0100 Subject: [PATCH 001/369] fix(upnp): the stale mapping cleanup works again, and the UPnP code survives a full router table nat-upnp 1.x fetches the router's device description before every action through the global http agent, which keeps sockets alive since Node 19. Routers close those sockets without saying so, so an action sent moments after another reuses a dead socket. The library hands the Error on as the description and the action throws "Cannot read properties of undefined (reading 'device')". The stale mapping cleanup lists the table one entry after another, so it died at entry 1, once per start, on every node behind such a router. Found on 8 nodes behind one TP-Link Archer VR2100v (libupnp 1.6.22) whose table sat full at 64 entries; reproduced there read-only: 1 entry then the TypeError, and with a socket per request all 64 in 452 ms. - the description fetch (1.x only) uses its own non-keepalive agent and rejects with its reason - refusals are logged with the router's UPnP code; an install that fails reports its port's reason, kept per port - a mapping the router refuses but already holds for this node, enabled, counts as mapped (GetSpecificPortMappingEntry), for app and core ports, so a full table no longer fails the refresh of mappings that work - removal looks the mapping up first and leaves alone one the router points at another machine (a sibling node behind the same router): a router removes by port alone; TCP and UDP are removed independently, 713/714 count as removed - a port that failed to map is no longer held from the cleanup, and the port test removes what the router kept of its refused mapping - the startup check makes room when its test mapping is refused: it removes the test mappings earlier starts left (nothing at a start holds one) and asks once more, instead of a node on a custom api port shutting down on a full table for good - every request this file sends the router has a timeout - the differences between nat-upnp 1.x and 2.x (local address, UPnP code on the error, wrapped fields, end-of-table code, gateway url) are kept to upnpService, so the move to 2.x needs no change here Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appInstaller.js | 4 +- .../appMonitoring/availabilityChecker.js | 3 +- ZelBack/src/services/upnpService.js | 406 ++++++++++++++---- tests/unit/availabilityChecker.test.js | 2 + tests/unit/upnpService.test.js | 356 ++++++++++++++- 5 files changed, 665 insertions(+), 106 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 6da5b03dab..f00c45a2b4 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -134,7 +134,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (res.flush) res.flush(); } } else { - throw new Error(`Error: Port ${port} FAILed to map.`); + throw new Error(`Error: Port ${port} FAILed to map. ${upnpService.getMapFailureReason(port)}`.trim()); } } } @@ -154,7 +154,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (res.flush) res.flush(); } } else { - throw new Error(`Error: Port ${appSpecifications.port} FAILed to map.`); + throw new Error(`Error: Port ${appSpecifications.port} FAILed to map. ${upnpService.getMapFailureReason(appSpecifications.port)}`.trim()); } } } diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index ae688e351a..5559b0b340 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -257,9 +257,10 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin dosState.lastUPNPMapFailed = true; log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} failed to create UPnP mapping`); setNextPort(); + // the router may have kept the TCP half: removing it leaves alone a + // half that points at another machine await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, - skipUpnp: true, skipHttpServer: true, }); const upnpDelay = dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error; diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 2650219861..2b3b1ab6b7 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -1,4 +1,7 @@ +const http = require('http'); const config = require('config'); +const axios = require('axios'); +const { XMLParser } = require('fast-xml-parser'); const natUpnp = require('@runonflux/nat-upnp'); const serviceHelper = require('./serviceHelper'); const messageHelper = require('./messageHelper'); @@ -10,7 +13,89 @@ const ufw = require('./utils/ufw'); const fluxadmPort = require('./fluxadmPort'); const { Privilege, authOf } = require('./utils/privileges'); -const client = new natUpnp.Client(); +// @runonflux/nat-upnp 2.x answers single-mapping lookups itself, takes a +// gateway url of its own and finds the local address itself; 1.x has none of +// that. What differs between the two is kept to this file. +const LIBRARY_V2 = typeof natUpnp.Client.prototype.getMapping === 'function'; +const { gatewayUrl } = config.upnp; +const gatewayNodeIp = config.upnp.nodeIp || '127.0.0.1'; +const client = gatewayUrl && LIBRARY_V2 + ? new natUpnp.Client({ url: gatewayUrl, localAddress: gatewayNodeIp }) + : new natUpnp.Client(); + +// nat-upnp 1.x fetches the router's device description before every action, +// through the global http agent, which keeps sockets alive since Node 19. +// Routers close those sockets without saying so, so an action sent moments +// after another reuses a dead socket and fails at once. The library then hands +// the Error on as if it were the description, and the action throws "Cannot +// read properties of undefined (reading 'device')", with the cause gone: the +// stale mapping cleanup, which lists the table one entry after another, died +// at its second entry. This is that one fetch, on a socket of its own, failing +// with its own reason. 2.x does both itself, and has no getXML to replace. +if (!LIBRARY_V2 && typeof natUpnp.Device.prototype.getXML === 'function') { + const descriptionAgent = new http.Agent({ keepAlive: false }); + natUpnp.Device.prototype.getXML = async function getXML(url) { + let data; + try { + ({ data } = await axios.get(url, { httpAgent: descriptionAgent, timeout: 10 * 1000, responseType: 'text' })); + } catch (error) { + throw new Error(`UPnP - the router's device description at ${url} could not be read: ${error.message}`); + } + const parsed = new XMLParser().parse(data); + if (!parsed || !parsed.root) throw new Error(`UPnP - ${url} is not a UPnP device description`); + return parsed; + }; +} + +// A router's own answer to a request: 713/714 past the end of the table or for +// a mapping it does not have (402 on some), 718 a port another machine holds, +// 728 a full table. +const UPNP_NO_SUCH_ENTRY = [713, 714]; +const UPNP_END_OF_TABLE = [402, 713, 714]; + +/** + * The UPnP error code of a router's refusal. 2.x throws it as `code`; 1.x + * leaves it in the SOAP fault of the reply. Null when the failure carried + * none: no reply at all, or a reply without a fault. + * @param {Error} error + * @returns {number|null} + */ +function upnpErrorCode(error) { + if (!error) return null; + if (Number.isInteger(error.code) && error.code > 0) return error.code; + const body = error.response && typeof error.response.data === 'string' ? error.response.data : ''; + const match = body.match(/<(?:\w+:)?errorCode>\s*(\d+)\s*([^<]*) String(value !== null && typeof value === 'object' && '#text' in value ? value['#text'] : value ?? ''); + +/** + * The gateway and the address this node has on its network, from either + * major version: 1.x hands the address over, 2.x is asked for it. + * @returns {Promise<{gateway: object, address: string}>} + */ +async function gatewayAndAddress() { + const info = await client.getGateway(); + const address = info.address !== undefined ? info.address : await info.getLocalAddress(); + return { gateway: info.gateway, address }; +} const FLUXADM_MAPPING_DESCRIPTION = 'Flux_Fluxadm_SSH'; @@ -75,17 +160,46 @@ async function withinTime(work, timeoutMs, what) { } } -if (config.upnp.gatewayUrl) { - // eslint-disable-next-line global-require - const { Device } = require('@runonflux/nat-upnp/build/src/nat-upnp/device'); - const { gatewayUrl } = config.upnp; - const nodeIp = config.upnp.nodeIp || '127.0.0.1'; +if (gatewayUrl && !LIBRARY_V2) { client.getGateway = async () => ({ - gateway: new Device(gatewayUrl), - address: nodeIp, + gateway: new natUpnp.Device(gatewayUrl), + address: gatewayNodeIp, }); } +// Every request this file sends the router itself. The library sets no timeout +// on its own requests: a router that takes a connection and never answers +// would hold an install, the port restore or the availability test forever. +const ROUTER_REQUEST_TIMEOUT_MS = 15 * 1000; +const toRouter = (work, what) => withinTime(work, ROUTER_REQUEST_TIMEOUT_MS, what); + +/** + * Who the router maps `port` over `protocol` to: 'here' (this node, port for + * port, enabled), 'disabled' (this node, but switched off), 'other' (another + * machine on the network - another Flux node behind the same router, the + * owner's own forward), 'none', or 'unknown' when the router could not be asked. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @returns {Promise} + */ +async function mappingOwner(port, protocol) { + try { + const { gateway, address } = await toRouter(gatewayAndAddress(), 'finding the gateway'); + const data = await toRouter(gateway.run('GetSpecificPortMappingEntry', [ + ['NewRemoteHost', ''], + ['NewExternalPort', String(port)], + ['NewProtocol', protocol], + ]), `reading the ${protocol} ${port} mapping`); + const key = Object.keys(data || {}).find((k) => /^GetSpecificPortMappingEntryResponse/.test(k)); + if (!key) return 'unknown'; + const entry = data[key]; + if (fieldText(entry.NewInternalClient) !== address || parseInt(fieldText(entry.NewInternalPort), 10) !== +port) return 'other'; + return fieldText(entry.NewEnabled) === '0' ? 'disabled' : 'here'; + } catch (error) { + return UPNP_NO_SUCH_ENTRY.includes(upnpErrorCode(error)) ? 'none' : 'unknown'; + } +} + let upnpMachine = false; /** @@ -131,6 +245,75 @@ async function adjustFirewallForUPNP() { } } +// What a FluxOS start leaves on the router and nothing at a start holds: the +// mappings of a port test, of an app's prelaunch test and of this check itself. +const TEST_MAPPING_DESCRIPTIONS = ['Flux_UPNP_Mapping_Test', 'Flux_Test_App']; +const TEST_MAPPING_PREFIX = 'Flux_Prelaunch_App_'; + +/** + * Removes the test mappings to this node that nothing here holds. Run when the + * router refuses this check's own mapping: on a router whose table is full the + * check fails on every start, a node on a custom api port shuts down on it, and + * the stale mapping cleanup, which runs later, is never reached. Test mappings + * are only ever made and removed by a test in progress, so at a start every + * one there is left over. + * @returns {Promise} how many were removed + */ +async function removeLeftoverTestMappings() { + const deadline = monotonicMs() + CLEANUP_SLOT_MS; + let mappings; + try { + mappings = await listOwnMappings(deadline - CLEANUP_REMOVAL_RESERVE_MS, deadline); + } catch (error) { + log.warn(`VerifyUPNPsupport - mappings not listed: ${describeUpnpError(error)}`); + return 0; + } + const leftovers = mappings.filter(({ description, public: { port } }) => typeof description === 'string' + && (TEST_MAPPING_DESCRIPTIONS.includes(description) || description.startsWith(TEST_MAPPING_PREFIX)) + && !mappedHere.has(port)); + let removed = 0; + // eslint-disable-next-line no-restricted-syntax + for (const { public: { host, port }, protocol, description } of leftovers) { + try { + // eslint-disable-next-line no-await-in-loop + await toRouter(client.removeMapping({ public: { host, port }, protocol: protocol.toUpperCase() }), 'removing a test mapping'); + removed += 1; + log.info(`VerifyUPNPsupport - left-over test mapping removed: ${protocol.toUpperCase()} ${port} (${description})`); + } catch (error) { + log.warn(`VerifyUPNPsupport - left-over test mapping ${protocol.toUpperCase()} ${port} not removed: ${describeUpnpError(error)}`); + } + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(500); + } + return removed; +} + +/** + * This check's own mapping. When the router refuses it: one an earlier start + * left is this node's already, and the removal that follows ends it; else the + * test mappings earlier starts left are removed to make room, and it is asked + * once more. + * @param {number} port + * @returns {Promise} + */ +async function createVerifyMapping(port) { + const mapping = { + public: port, + private: port, + ttl: 0, + description: 'Flux_UPNP_Mapping_Test', + }; + try { + await toRouter(client.createMapping(mapping), 'creating the test mapping'); + } catch (error) { + log.warn(`VerifyUPNPsupport - test mapping refused: ${describeUpnpError(error)}`); + await serviceHelper.delay(500); + if (await mappingOwner(port, 'TCP') === 'here') return; + if (!await removeLeftoverTestMappings()) throw error; + await toRouter(client.createMapping(mapping), 'creating the test mapping'); + } +} + /** * To verify that a port has UPnP (Universal Plug and Play) support. * @param {number} apiport Port number. @@ -146,8 +329,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { await serviceHelper.delay(500); } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed get public ip'); + log.error(`VerifyUPNPsupport - Failed get public ip: ${describeUpnpError(error)}`); upnpMachine = false; return false; } @@ -156,8 +338,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { await serviceHelper.delay(500); } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed get Gateway'); + log.error(`VerifyUPNPsupport - Failed get Gateway: ${describeUpnpError(error)}`); upnpMachine = false; return false; } @@ -166,17 +347,11 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { // router IP exits on that. A test that fails leaves it held. mappedHere.add(+apiport + 3); try { - await client.createMapping({ - public: +apiport + 3, - private: +apiport + 3, - ttl: 0, - description: 'Flux_UPNP_Mapping_Test', - }); + await createVerifyMapping(+apiport + 3); await serviceHelper.delay(500); } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed Create Mapping'); + log.error(`VerifyUPNPsupport - Failed Create Mapping: ${describeUpnpError(error)}`); upnpMachine = false; return false; } @@ -185,8 +360,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { await serviceHelper.delay(500); } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed get Mappings'); + log.error(`VerifyUPNPsupport - Failed get Mappings: ${describeUpnpError(error)}`); upnpMachine = false; return false; } @@ -197,8 +371,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { await serviceHelper.delay(500); } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed Remove Mapping'); + log.error(`VerifyUPNPsupport - Failed Remove Mapping: ${describeUpnpError(error)}`); upnpMachine = false; return false; } @@ -255,39 +428,19 @@ async function reconcileFluxadmMapping(apiport) { */ async function setupUPNP(apiport = config.server.apiport) { try { - await client.createMapping({ - public: +apiport, - private: +apiport, - ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes - description: 'Flux_Backend_API', - }); + await mapCorePort(+apiport, 'Flux_Backend_API'); await serviceHelper.delay(500); - await client.createMapping({ - public: +apiport + 1, - private: +apiport + 1, - ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes - description: 'Flux_Backend_API_SSL', - }); + await mapCorePort(+apiport + 1, 'Flux_Backend_API_SSL'); await serviceHelper.delay(500); - await client.createMapping({ - public: +apiport - 1, - private: +apiport - 1, - ttl: 0, - description: 'Flux_Home_UI', - }); + await mapCorePort(+apiport - 1, 'Flux_Home_UI'); await serviceHelper.delay(500); - await client.createMapping({ - public: +apiport + 2, - private: +apiport + 2, - ttl: 0, - description: 'Flux_Syncthing', - }); + await mapCorePort(+apiport + 2, 'Flux_Syncthing'); await serviceHelper.delay(500); @@ -295,13 +448,79 @@ async function setupUPNP(apiport = config.server.apiport) { return true; } catch (error) { - log.error(error); + log.error(`setupUPNP - ${describeUpnpError(error)}`); return false; } } +// Why the router refused each port's last refused mapping, for an install to report. +const mapFailureReasons = new Map(); + +/** + * Why the router refused the last mapping of `port`, as mapUpnpPort logged it. + * @param {number} port + * @returns {string} + */ +function getMapFailureReason(port) { + return mapFailureReasons.get(+port) || ''; +} + +/** + * Maps one protocol of a port. A refusal is no failure when the router already + * maps the port here: some routers refuse to add an entry that exists, and a + * full table refuses everything, the refresh of a mapping that works included. + * @param {number} port + * @param {string} protocol + * @param {string} description + * @returns {Promise} + */ +async function mapProtocol(port, protocol, description) { + try { + await toRouter(client.createMapping({ + public: port, + private: port, + ttl: 0, + protocol, + description, + }), `mapping ${protocol} ${port}`); + } catch (error) { + const reason = describeUpnpError(error); + await serviceHelper.delay(500); + if (await mappingOwner(port, protocol) === 'here') { + log.info(`UPnP - ${protocol} ${port} refused (${reason}), but the router already maps it to this node`); + return; + } + throw new Error(`UPnP - ${protocol} ${port} not mapped: ${reason}`); + } +} + +/** + * Maps one of FluxOS's own ports, TCP: the api, its SSL port, the home UI, + * syncthing. One the router already maps here is mapped. + * @param {number} port + * @param {string} description + * @returns {Promise} + */ +async function mapCorePort(port, description) { + try { + await toRouter(client.createMapping({ + public: port, + private: port, + ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes + description, + }), `mapping ${port}`); + } catch (error) { + await serviceHelper.delay(500); + if (await mappingOwner(port, 'TCP') !== 'here') throw error; + log.info(`setupUPNP - ${port} refused (${describeUpnpError(error)}), but the router already maps it to this node`); + } +} + /** * To create mappings for UPnP (Universal Plug and Play) port. + * + * On a failure the port is no longer held here: whatever half of it the router + * kept is the caller's to remove, or the stale mapping cleanup's. * @param {number} port Port number. * @param {string} description Port description. * @returns {Promise} True if port mappings can be created for both TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) protocols. Otherwise false. @@ -310,62 +529,64 @@ async function mapUpnpPort(port, description) { // held from before the router has it: the cleanup pass may list it the // moment it is there mappedHere.add(+port); + mapFailureReasons.delete(+port); try { - await client.createMapping({ - public: port, - private: port, - ttl: 0, - protocol: 'TCP', - description, - }); + await mapProtocol(port, 'TCP', description); await serviceHelper.delay(500); - await client.createMapping({ - public: port, - private: port, - ttl: 0, - protocol: 'UDP', - description, - }); + await mapProtocol(port, 'UDP', description); await serviceHelper.delay(500); return true; } catch (error) { - log.error(error); + log.error(error.message); + mapFailureReasons.set(+port, error.message); + mappedHere.delete(+port); return false; } } /** * To remove TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) port mappings from UPnP (Universal Plug and Play) port. + * + * A router removes a mapping by port alone, whoever it points at, so each + * protocol is looked up first: one that points at another machine - another + * Flux node behind the same router, whose port this node failed to map - is + * left alone, one the router does not have is skipped. Each protocol is + * removed on its own, a refusal of one leaving the other to go. * @param {number} port Port number. * @returns {Promise} True if port mappings have been removed for both TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) protocols. Otherwise false. */ async function removeMapUpnpPort(port) { // given up even if the router refuses: what is left is the cleanup pass's mappedHere.delete(+port); - try { - await client.removeMapping({ - public: port, - protocol: 'TCP', - }); - - await serviceHelper.delay(500); - - await client.removeMapping({ - public: port, - protocol: 'UDP', - }); - + let removedBoth = true; + // eslint-disable-next-line no-restricted-syntax + for (const protocol of ['TCP', 'UDP']) { + // eslint-disable-next-line no-await-in-loop + const owner = await mappingOwner(port, protocol); + if (owner === 'other') { + log.info(`UPnP - ${protocol} ${port} not unmapped: the router maps it to another machine`); + } else if (owner !== 'none') { + try { + // eslint-disable-next-line no-await-in-loop + await toRouter(client.removeMapping({ + public: port, + protocol, + }), `unmapping ${protocol} ${port}`); + } catch (error) { + if (!UPNP_NO_SUCH_ENTRY.includes(upnpErrorCode(error))) { + removedBoth = false; + log.error(`UPnP - ${protocol} ${port} not unmapped: ${describeUpnpError(error)}`); + } + } + } + // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(500); - - return true; - } catch (error) { - log.error(error); - return false; } + return removedBoth; } /** @@ -419,7 +640,7 @@ function msUntilCleanupSlot(apiport, now = Date.now()) { */ async function listOwnMappings(until, deadline) { const timeout = () => Math.min(CLEANUP_REQUEST_TIMEOUT_MS, deadline - monotonicMs()); - const { gateway, address } = await withinTime(client.getGateway(), timeout(), 'finding the gateway'); + const { gateway, address } = await withinTime(gatewayAndAddress(), timeout(), 'finding the gateway'); const mappings = []; for (let index = 0; ; index += 1) { if (monotonicMs() >= until) { @@ -431,18 +652,19 @@ async function listOwnMappings(until, deadline) { // eslint-disable-next-line no-await-in-loop data = await withinTime(gateway.run('GetGenericPortMappingEntry', [['NewPortMappingIndex', index]]), timeout(), 'listing the mappings'); } catch (error) { - if (error.response) return mappings; + // 1.x rejects a fault as an HTTP error, 2.x with its UPnP code + if (error.response || UPNP_END_OF_TABLE.includes(upnpErrorCode(error))) return mappings; throw error; } const key = Object.keys(data || {}).find((k) => /^GetGenericPortMappingEntryResponse/.test(k)); if (!key) throw new Error('listing the mappings: incorrect response'); const entry = data[key]; - if (entry.NewInternalClient === address) { + if (fieldText(entry.NewInternalClient) === address) { mappings.push({ - public: { host: (typeof entry.NewRemoteHost === 'string' && entry.NewRemoteHost) || '', port: parseInt(entry.NewExternalPort, 10) }, - private: { host: entry.NewInternalClient, port: parseInt(entry.NewInternalPort, 10) }, - protocol: String(entry.NewProtocol).toLowerCase(), - description: entry.NewPortMappingDescription, + public: { host: fieldText(entry.NewRemoteHost), port: parseInt(fieldText(entry.NewExternalPort), 10) }, + private: { host: fieldText(entry.NewInternalClient), port: parseInt(fieldText(entry.NewInternalPort), 10) }, + protocol: fieldText(entry.NewProtocol).toLowerCase(), + description: fieldText(entry.NewPortMappingDescription), }); } } @@ -501,7 +723,7 @@ async function removeStaleMappings(heldPorts, budgetMs) { removed += 1; log.info(`UPnP - stale mapping removed: ${protocol} ${port} (${mapping.description})`); } catch (error) { - log.warn(`UPnP - stale mapping ${protocol} ${port} (${mapping.description}) not removed: ${error.message}`); + log.warn(`UPnP - stale mapping ${protocol} ${port} (${mapping.description}) not removed: ${describeUpnpError(error)}`); if (error.timedOut) { log.warn(`UPnP - stale mapping cleanup stopped, ${stale.length - index - 1} left for the next start`); break; @@ -687,6 +909,10 @@ module.exports = { setupUPNP, mapUpnpPort, removeMapUpnpPort, + getMapFailureReason, + describeUpnpError, + upnpErrorCode, + mappingOwner, corePorts, msUntilCleanupSlot, removeStaleMappings, diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 255b111563..f563170cc7 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -351,6 +351,8 @@ describe('availabilityChecker tests', () => { ); expect(mockDosState.lastUPNPMapFailed).to.be.true; + // the router may have kept the TCP half of the refused mapping + sinon.assert.calledOnceWithExactly(upnpService.removeMapUpnpPort, mockDosState.testingPort, 'Flux_Test_App'); }); it('opens the port under test to IPv4 only, and deletes only that rule', async () => { diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index c73ca90245..e73f473a88 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -18,6 +18,23 @@ const batches = (stub) => stub.getCalls() const { expect } = chai; +// A router's refusal as nat-upnp 1.x rejects it: an HTTP 500 with the SOAP fault in the body. +const soapFault = (code, description) => { + const error = new Error('Request failed with status code 500'); + error.response = { + status: 500, + data: `${code}` + + `${description}`, + }; + return error; +}; +// The router's answer to GetSpecificPortMappingEntry. +const entryFor = (internalClient, port, enabled = '1') => ({ + GetSpecificPortMappingEntryResponse: { NewInternalClient: internalClient, NewInternalPort: String(port), NewEnabled: enabled }, +}); +// The gateway as nat-upnp 1.x hands it over, this node at 192.168.1.10. +const stubGateway = (run) => sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves({ gateway: { run }, address: '192.168.1.10' }); + const config = { apiport: '5550', }; @@ -162,8 +179,8 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get public ip'); + sinon.assert.calledOnce(logSpy); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get public ip: Error'); }); it('should log a proper error if getGateway throws', async () => { @@ -181,8 +198,8 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Gateway'); + sinon.assert.calledOnce(logSpy); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Gateway: Error'); }); it('should log a proper error if createMapping throws', async () => { @@ -200,8 +217,8 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping'); + sinon.assert.calledOnce(logSpy); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping: Error'); }); it('should log a proper error if getMappings throws', async () => { @@ -219,8 +236,8 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mappings'); + sinon.assert.calledOnce(logSpy); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mappings: Error'); }); it('should log a proper error if removeMapping throws', async () => { @@ -238,8 +255,78 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Remove Mapping'); + sinon.assert.calledOnce(logSpy); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Remove Mapping: Error'); + }); + + it('should make room when the router refuses its test mapping: remove the test mappings earlier starts left, then ask again', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves('1.2.3.4'); + sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + const createStub = sinon.stub(natUpnp.Client.prototype, 'createMapping'); + createStub.onFirstCall().rejects(soapFault(728, 'NoPortMapsAvailable')); + createStub.resolves(); + const removeStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + const table = [ + ['192.168.1.10', 41713, 'TCP', 'Flux_Test_App'], + ['192.168.1.10', 30001, 'TCP', 'Flux_App_kept'], + ['192.168.1.11', 39988, 'TCP', 'Flux_Test_App'], + ['192.168.1.10', 52217, 'UDP', 'Flux_Prelaunch_App_52217'], + ]; + const run = sinon.stub().callsFake(async (action, args) => { + if (action === 'GetSpecificPortMappingEntry') throw soapFault(714, 'NoSuchEntryInArray'); + const index = args[0][1]; + if (index >= table.length) throw soapFault(713, 'SpecifiedArrayIndexInvalid'); + const [internalClient, port, protocol, description] = table[index]; + return { + GetGenericPortMappingEntryResponse: { + NewRemoteHost: '', NewExternalPort: String(port), NewProtocol: protocol, NewInternalClient: internalClient, NewInternalPort: String(port), NewPortMappingDescription: description, + }, + }; + }); + stubGateway(run); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(10_000); + const result = await promise; + + expect(result).to.equal(true); + sinon.assert.calledTwice(createStub); + // this node's test mappings go; its app's and another machine's stay + sinon.assert.calledWithExactly(removeStub, { public: { host: '', port: 41713 }, protocol: 'TCP' }); + sinon.assert.calledWithExactly(removeStub, { public: { host: '', port: 52217 }, protocol: 'UDP' }); + sinon.assert.neverCalledWithMatch(removeStub, { public: { port: 30001 } }); + sinon.assert.neverCalledWithMatch(removeStub, { public: { port: 39988 } }); + }); + + it('should go on when the refused test mapping is one an earlier start left for this node', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves('1.2.3.4'); + sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + const createStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').rejects(soapFault(718, 'ConflictInMappingEntry')); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + stubGateway(sinon.stub().resolves(entryFor('192.168.1.10', 16130))); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(10_000); + + expect(await promise).to.equal(true); + sinon.assert.calledOnce(createStub); + }); + + it('should still fail when the router refuses and nothing can be removed', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves('1.2.3.4'); + sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + sinon.stub(natUpnp.Client.prototype, 'createMapping').rejects(soapFault(728, 'NoPortMapsAvailable')); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + stubGateway(sinon.stub().rejects(soapFault(713, 'SpecifiedArrayIndexInvalid'))); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(10_000); + + expect(await promise).to.equal(false); + sinon.assert.calledOnceWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping: UPnP error 728 NoPortMapsAvailable (HTTP 500)'); }); }); @@ -261,6 +348,7 @@ describe('upnpService tests', () => { createMappingSpy = sinon.stub(natUpnp.Client.prototype, 'createMapping'); getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + stubGateway(sinon.stub().rejects(soapFault(714, 'NoSuchEntryInArray'))); }); afterEach(() => { @@ -429,17 +517,45 @@ describe('upnpService tests', () => { expect(result).to.equal(false); sinon.assert.calledOnce(logSpy); }); + + it('should take a core port the router refuses but already maps to this node as mapped', async () => { + natUpnp.Client.prototype.getGateway.restore(); + stubGateway(sinon.stub().callsFake(async (action, args) => entryFor('192.168.1.10', args[1][1]))); + createMappingSpy.returns(true); + createMappingSpy.withArgs(sinon.match({ description: 'Flux_Backend_API' })).rejects(soapFault(728, 'NoPortMapsAvailable')); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.setupUPNP(16127); + await clock.tickAsync(5_000); + const result = await promise; + + expect(result).to.equal(true); + sinon.assert.notCalled(logSpy); + }); }); describe('mapUpnpPort tests', () => { let logSpy; let createMappingSpy; + let runStub; + beforeEach(() => { logSpy = sinon.spy(log, 'error'); createMappingSpy = sinon.stub(natUpnp.Client.prototype, 'createMapping'); + runStub = sinon.stub().rejects(soapFault(714, 'NoSuchEntryInArray')); + stubGateway(runStub); }); + async function runMap(port) { + const clock = sinon.useFakeTimers(); + const promise = upnpService.mapUpnpPort(port, 'Flux_App_test'); + await clock.tickAsync(3_000); + const result = await promise; + clock.restore(); + return result; + } + afterEach(() => { sinon.restore(); }); @@ -486,17 +602,189 @@ describe('upnpService tests', () => { expect(result).to.equal(false); sinon.assert.calledOnce(logSpy); }); + + it('should log the router\'s UPnP code, and keep it as the port\'s reason for an install to report', async () => { + createMappingSpy.rejects(soapFault(728, 'NoPortMapsAvailable')); + + expect(await runMap(123)).to.equal(false); + sinon.assert.calledOnceWithExactly(logSpy, 'UPnP - TCP 123 not mapped: UPnP error 728 NoPortMapsAvailable (HTTP 500)'); + expect(upnpService.getMapFailureReason(123)).to.equal('UPnP - TCP 123 not mapped: UPnP error 728 NoPortMapsAvailable (HTTP 500)'); + expect(upnpService.getMapFailureReason(124)).to.equal(''); + }); + + it('should take a refused mapping the router already holds for this node as mapped', async () => { + createMappingSpy.rejects(soapFault(718, 'ConflictInMappingEntry')); + runStub.resolves(entryFor('192.168.1.10', 123)); + + expect(await runMap(123)).to.equal(true); + sinon.assert.notCalled(logSpy); + sinon.assert.calledWith(runStub, 'GetSpecificPortMappingEntry', [['NewRemoteHost', ''], ['NewExternalPort', '123'], ['NewProtocol', 'TCP']]); + }); + + it('should not take a refused mapping held for another machine, or switched off, as mapped', async () => { + createMappingSpy.rejects(soapFault(718, 'ConflictInMappingEntry')); + runStub.resolves(entryFor('192.168.1.11', 123)); + expect(await runMap(123)).to.equal(false); + + runStub.resolves(entryFor('192.168.1.10', 123, '0')); + expect(await runMap(123)).to.equal(false); + }); + + it('should stop holding a port it failed to map, so the cleanup can take what the router kept', async () => { + createMappingSpy.withArgs(sinon.match({ protocol: 'TCP' })).resolves(); + createMappingSpy.withArgs(sinon.match({ protocol: 'UDP' })).rejects(soapFault(728, 'NoPortMapsAvailable')); + + expect(await runMap(41713)).to.equal(false); + expect(upnpService.mappedHere.has(41713)).to.equal(false); + }); + + it('should give up on a router that never answers', async () => { + createMappingSpy.returns(new Promise(() => {})); + runStub.returns(new Promise(() => {})); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.mapUpnpPort(123, 'Flux_App_test'); + await clock.tickAsync(40_000); + + expect(await promise).to.equal(false); + sinon.assert.calledOnceWithMatch(logSpy, /^UPnP - TCP 123 not mapped: mapping TCP 123: the router did not answer within 15s$/); + }); + }); + + describe('describeUpnpError tests', () => { + it('should name the UPnP code and the router\'s wording from a 1.x SOAP fault', () => { + const error = soapFault(718, 'ConflictInMappingEntry'); + + expect(upnpService.describeUpnpError(error)).to.equal('UPnP error 718 ConflictInMappingEntry (HTTP 500)'); + expect(upnpService.upnpErrorCode(error)).to.equal(718); + }); + + it('should read the code 2.x throws, and its message', () => { + const error = new Error('UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)'); + error.code = 728; + + expect(upnpService.upnpErrorCode(error)).to.equal(728); + expect(upnpService.describeUpnpError(error)).to.equal('UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)'); + }); + + it('should fall back to the message when the failure carried no code', () => { + const error = new Error('socket hang up'); + error.code = 'ECONNRESET'; + + expect(upnpService.describeUpnpError(error)).to.equal('socket hang up'); + expect(upnpService.upnpErrorCode(error)).to.equal(null); + }); + }); + + describe('mappingOwner tests', () => { + afterEach(() => { + sinon.restore(); + }); + + it('should ask 2.x for the local address it no longer hands over', async () => { + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves({ + gateway: { run: sinon.stub().resolves(entryFor('192.168.1.10', 123)) }, + getLocalAddress: sinon.stub().resolves('192.168.1.10'), + }); + + expect(await upnpService.mappingOwner(123, 'TCP')).to.equal('here'); + }); + + it('should read fields 2.x\'s parser wraps', async () => { + stubGateway(sinon.stub().resolves({ + GetSpecificPortMappingEntryResponse: { NewInternalClient: { '#text': '192.168.1.10' }, NewInternalPort: 123, NewEnabled: 1 }, + })); + + expect(await upnpService.mappingOwner(123, 'TCP')).to.equal('here'); + }); + + it('should tell no mapping from a router that could not be asked', async () => { + stubGateway(sinon.stub().rejects(soapFault(714, 'NoSuchEntryInArray'))); + expect(await upnpService.mappingOwner(123, 'TCP')).to.equal('none'); + + natUpnp.Client.prototype.getGateway.restore(); + stubGateway(sinon.stub().rejects(new Error('socket hang up'))); + expect(await upnpService.mappingOwner(123, 'TCP')).to.equal('unknown'); + }); + }); + + describe('device description fetch tests', () => { + // eslint-disable-next-line global-require + const axios = require('axios'); + const url = 'http://192.168.1.1:1900/gatedesc.xml'; + + afterEach(() => { + sinon.restore(); + }); + + it('should fetch on a socket of its own, never a kept-alive one', async () => { + const getStub = sinon.stub(axios, 'get').resolves({ data: 'x' }); + + const parsed = await new natUpnp.Device(url).getXML(url); + + expect(parsed.root.device.deviceType).to.equal('x'); + expect(getStub.firstCall.args[1].httpAgent.keepAlive).to.equal(false); + }); + + it('should reject with the reason when the description cannot be read', async () => { + sinon.stub(axios, 'get').rejects(new Error('socket hang up')); + + const result = await new natUpnp.Device(url).getXML(url).catch((error) => error); + + expect(result.message).to.equal(`UPnP - the router's device description at ${url} could not be read: socket hang up`); + }); + + it('should reject a reply that is not a device description', async () => { + sinon.stub(axios, 'get').resolves({ data: 'login' }); + + const result = await new natUpnp.Device(url).getXML(url).catch((error) => error); + + expect(result.message).to.equal(`UPnP - ${url} is not a UPnP device description`); + }); + + it('should resolve a real router description through to the action it sends', async () => { + sinon.stub(axios, 'get').resolves({ + data: 'http://192.168.1.1:1900' + + 'urn:schemas-upnp-org:device:InternetGatewayDevice:1' + + 'urn:schemas-upnp-org:device:WANDevice:1' + + 'urn:schemas-upnp-org:device:WANConnectionDevice:1' + + 'urn:schemas-upnp-org:service:WANIPConnection:1urn:upnp-org:serviceId:WANIPConn1' + + '/upnp/control/WANIPConn1/upnp/event/WANIPConn1/gateconnSCPD.xml' + + '', + }); + + const service = await new natUpnp.Device(url).getService(['urn:schemas-upnp-org:service:WANIPConnection:1']); + + expect(service).to.deep.equal({ + service: 'urn:schemas-upnp-org:service:WANIPConnection:1', + SCPDURL: 'http://192.168.1.1:1900/gateconnSCPD.xml', + controlURL: 'http://192.168.1.1:1900/upnp/control/WANIPConn1', + }); + }); }); describe('removeMapUpnpPort tests', () => { let logSpy; let removeMappingSpy; + let runStub; + beforeEach(() => { logSpy = sinon.spy(log, 'error'); removeMappingSpy = sinon.stub(natUpnp.Client.prototype, 'removeMapping'); + runStub = sinon.stub().callsFake(async (action, args) => entryFor('192.168.1.10', args[1][1])); + stubGateway(runStub); }); + async function runRemove(port) { + const clock = sinon.useFakeTimers(); + const promise = upnpService.removeMapUpnpPort(port); + await clock.tickAsync(3_000); + const result = await promise; + clock.restore(); + return result; + } + afterEach(() => { sinon.restore(); }); @@ -508,7 +796,7 @@ describe('upnpService tests', () => { const promise = upnpService.removeMapUpnpPort(123, 'some description'); - await clock.tickAsync(1_000); + await clock.tickAsync(3_000); const result = await promise; expect(result).to.equal(true); @@ -525,11 +813,41 @@ describe('upnpService tests', () => { const promise = upnpService.removeMapUpnpPort(123); - await clock.tickAsync(1_000); + await clock.tickAsync(3_000); const result = await promise; expect(result).to.equal(false); - sinon.assert.calledOnce(logSpy); + // the UDP half is still tried when TCP is refused + sinon.assert.calledTwice(removeMappingSpy); + sinon.assert.calledTwice(logSpy); + }); + + it('should leave alone a mapping the router points at another machine', async () => { + removeMappingSpy.resolves(); + runStub.callsFake(async (action, args) => (args[2][1] === 'UDP' ? entryFor('192.168.1.11', 123) : entryFor('192.168.1.10', 123))); + + expect(await runRemove(123)).to.equal(true); + sinon.assert.calledOnceWithExactly(removeMappingSpy, { public: 123, protocol: 'TCP' }); + }); + + it('should skip a mapping the router does not have, and count one gone by the time it is removed', async () => { + removeMappingSpy.rejects(soapFault(714, 'NoSuchEntryInArray')); + runStub.callsFake(async (action, args) => { + if (args[2][1] === 'TCP') throw soapFault(714, 'NoSuchEntryInArray'); + return entryFor('192.168.1.10', 123); + }); + + expect(await runRemove(123)).to.equal(true); + sinon.assert.calledOnceWithExactly(removeMappingSpy, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(logSpy); + }); + + it('should still remove when the router cannot say who holds the port', async () => { + removeMappingSpy.resolves(); + runStub.rejects(new Error('socket hang up')); + + expect(await runRemove(123)).to.equal(true); + sinon.assert.calledTwice(removeMappingSpy); }); }); @@ -619,6 +937,18 @@ describe('upnpService tests', () => { sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP' }); }); + it('should end the walk on the end-of-table code 2.x throws, which carries no HTTP reply', async () => { + routerHas([mapping(31000, 'Flux_App_gone')]); + runStub.callsFake(async (action, [[, index]]) => { + if (index >= table.length) throw Object.assign(new Error('UPnP error 713: SpecifiedArrayIndexInvalid'), { code: 713 }); + return asEntry(table[index]); + }); + + const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); + + expect(removed).to.equal(1); + }); + it('should take an error reply at the first index for an empty table', async () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); From 56942d19a34fba61125933b236835ebffc1c2c58 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 14:18:55 +0100 Subject: [PATCH 002/369] chore(upnp): move to @runonflux/nat-upnp 2.0.0 The stale-mapping cleanup lists the router's table with the library's getMappings({ local: true }) and bounds the listing and each removal with an AbortSignal, which cancels the request at the router. A listing that does not finish within its share of the slot removes nothing. A removal the router refuses with a UPnP error is stepped past; any other failure ends the pass. config.upnp.gatewayUrl and nodeIp configure the client directly. /flux/getgateway answers with the same shape as before: { gateway: { description, services }, address }. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 121 ++++--------------- package.json | 2 +- tests/unit/upnpService.test.js | 180 +++++++++++++--------------- 3 files changed, 109 insertions(+), 194 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 2650219861..5893cba80d 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -10,7 +10,11 @@ const ufw = require('./utils/ufw'); const fluxadmPort = require('./fluxadmPort'); const { Privilege, authOf } = require('./utils/privileges'); -const client = new natUpnp.Client(); +// config.upnp.gatewayUrl names the router's device description, used in place +// of discovering the router; the node's address is then config.upnp.nodeIp. +const client = config.upnp.gatewayUrl + ? new natUpnp.Client({ url: config.upnp.gatewayUrl, localAddress: config.upnp.nodeIp || '127.0.0.1' }) + : new natUpnp.Client(); const FLUXADM_MAPPING_DESCRIPTION = 'Flux_Fluxadm_SSH'; @@ -40,52 +44,12 @@ const mappedHere = new Set(); const CLEANUP_CYCLE_MS = 10 * 60 * 1000; const CLEANUP_SLOT_COUNT = 8; const CLEANUP_SLOT_MS = CLEANUP_CYCLE_MS / CLEANUP_SLOT_COUNT; -// The library's requests have no timeout of their own: a router that restarts -// with one in flight never answers it. -const CLEANUP_REQUEST_TIMEOUT_MS = 15 * 1000; -// The listing stops asking this long before the slot ends, so that what it has -// listed by then still has time to be removed. +// The listing has to finish this long before the slot ends, so that what it +// lists still has time to be removed. const CLEANUP_REMOVAL_RESERVE_MS = 15 * 1000; const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); -/** - * Settles with `work`, or rejects once `timeoutMs` has passed, with an error - * marked `timedOut`. Only the wait ends: the request is still in flight. - * @param {Promise<*>} work - * @param {number} timeoutMs - * @param {string} what what is waited for, for the error - * @returns {Promise<*>} - */ -async function withinTime(work, timeoutMs, what) { - let timer; - try { - return await Promise.race([ - work, - new Promise((_, reject) => { - timer = setTimeout(() => { - const error = new Error(`${what}: the router did not answer within ${timeoutMs / 1000}s`); - error.timedOut = true; - reject(error); - }, timeoutMs); - }), - ]); - } finally { - clearTimeout(timer); - } -} - -if (config.upnp.gatewayUrl) { - // eslint-disable-next-line global-require - const { Device } = require('@runonflux/nat-upnp/build/src/nat-upnp/device'); - const { gatewayUrl } = config.upnp; - const nodeIp = config.upnp.nodeIp || '127.0.0.1'; - client.getGateway = async () => ({ - gateway: new Device(gatewayUrl), - address: nodeIp, - }); -} - let upnpMachine = false; /** @@ -401,53 +365,6 @@ function msUntilCleanupSlot(apiport, now = Date.now()) { return (slotStart - (now % CLEANUP_CYCLE_MS) + CLEANUP_CYCLE_MS) % CLEANUP_CYCLE_MS; } -/** - * Lists the router's mappings to this node, one GetGenericPortMappingEntry per - * index, as the library's getMappings does. The walk is this code's rather than - * the library's so that it can stop: getMappings walks to the end of the table - * whatever its caller has given up on, into the next node's slot. No request - * goes out once `until` has passed (one already in flight cannot be called - * back under nat-upnp 1.x), and what was listed by then is returned: the pass - * only removes what it has seen. - * - * The router's error reply to an index past the last entry ends the table. No - * reply at all - a request that times out, a connection refused or reset - is a - * router in trouble, and rejects: nothing more is sent to it. - * @param {number} until monotonic ms after which no request is sent - * @param {number} deadline monotonic ms the whole pass ends at - * @returns {Promise} the mappings, in the library's getMappings shape - */ -async function listOwnMappings(until, deadline) { - const timeout = () => Math.min(CLEANUP_REQUEST_TIMEOUT_MS, deadline - monotonicMs()); - const { gateway, address } = await withinTime(client.getGateway(), timeout(), 'finding the gateway'); - const mappings = []; - for (let index = 0; ; index += 1) { - if (monotonicMs() >= until) { - log.warn(`UPnP - stale mapping cleanup out of time after ${index} router entries, the rest wait for the next start`); - return mappings; - } - let data; - try { - // eslint-disable-next-line no-await-in-loop - data = await withinTime(gateway.run('GetGenericPortMappingEntry', [['NewPortMappingIndex', index]]), timeout(), 'listing the mappings'); - } catch (error) { - if (error.response) return mappings; - throw error; - } - const key = Object.keys(data || {}).find((k) => /^GetGenericPortMappingEntryResponse/.test(k)); - if (!key) throw new Error('listing the mappings: incorrect response'); - const entry = data[key]; - if (entry.NewInternalClient === address) { - mappings.push({ - public: { host: (typeof entry.NewRemoteHost === 'string' && entry.NewRemoteHost) || '', port: parseInt(entry.NewExternalPort, 10) }, - private: { host: entry.NewInternalClient, port: parseInt(entry.NewInternalPort, 10) }, - protocol: String(entry.NewProtocol).toLowerCase(), - description: entry.NewPortMappingDescription, - }); - } - } -} - /** * Removes the router's mappings to this node that this code made and nothing * here holds any more: an app removed while the router was unreachable, a @@ -466,10 +383,11 @@ async function listOwnMappings(until, deadline) { * holds already has its app in the answer. * * Everything - the listing and each removal - happens within `budgetMs`. The - * listing stops asking the router early enough to leave time for removals, and - * what has not been listed or removed by the end is left for the next start. A - * removal the router refuses is stepped past; one it does not answer ends the - * pass, so as not to send more to a router that is not keeping up. + * listing has to finish early enough to leave time for removals, or nothing is + * removed; what has not been removed by the end is left for the next start. A + * removal the router refuses with a UPnP error is stepped past; any other + * failure - no answer in time, a connection refused or reset - ends the pass, + * so as not to send more to a router that is not keeping up. * @param {function(): Promise>} heldPorts every port something on this node holds * @param {number} budgetMs * @returns {Promise} how many mappings were removed @@ -477,7 +395,8 @@ async function listOwnMappings(until, deadline) { async function removeStaleMappings(heldPorts, budgetMs) { const deadline = monotonicMs() + budgetMs; const left = () => deadline - monotonicMs(); - const mappings = await listOwnMappings(deadline - Math.min(CLEANUP_REMOVAL_RESERVE_MS, budgetMs / 2), deadline); + const listingMs = budgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, budgetMs / 2); + const mappings = await client.getMappings({ local: true, signal: AbortSignal.timeout(listingMs) }); const keep = new Set([...await heldPorts()].map(Number)); const stale = mappings.filter((mapping) => Number.isInteger(mapping.public.port) && !keep.has(mapping.public.port) @@ -497,12 +416,12 @@ async function removeStaleMappings(heldPorts, budgetMs) { const protocol = mapping.protocol.toUpperCase(); try { // eslint-disable-next-line no-await-in-loop - await withinTime(client.removeMapping({ public: { host, port }, protocol }), Math.min(CLEANUP_REQUEST_TIMEOUT_MS, left()), 'removing the mapping'); + await client.removeMapping({ public: { host, port }, protocol, signal: AbortSignal.timeout(left()) }); removed += 1; log.info(`UPnP - stale mapping removed: ${protocol} ${port} (${mapping.description})`); } catch (error) { log.warn(`UPnP - stale mapping ${protocol} ${port} (${mapping.description}) not removed: ${error.message}`); - if (error.timedOut) { + if (!(error instanceof natUpnp.UpnpError)) { log.warn(`UPnP - stale mapping cleanup stopped, ${stale.length - index - 1} left for the next start`); break; } @@ -663,8 +582,12 @@ async function getGatewayApi(req, res) { try { const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { - const gateway = await client.getGateway(); - const message = messageHelper.createDataMessage(gateway); + const info = await client.getGateway(); + const { description, services } = info.gateway; + const message = messageHelper.createDataMessage({ + gateway: { description, services }, + address: await info.getLocalAddress(), + }); res.json(message); } else { const errMessage = messageHelper.errUnauthorizedMessage(); diff --git a/package.json b/package.json index bbb2cff7b6..934e8ff1bf 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "@aws-sdk/client-ecr": "^3.922.0", "@azure/identity": "^4.13.0", "@isaacs/ttlcache": "~1.4.1", - "@runonflux/nat-upnp": "~1.0.2", + "@runonflux/nat-upnp": "~2.0.0", "apicache": "~1.6.3", "archiver": "~7.0.1", "axios": "~1.13.6", diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index c73ca90245..d889cbf643 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -44,6 +44,12 @@ const upnpService = proxyquire( { config, './fluxadmPort': fluxadmPortStub }, ); +const GATEWAY_URL = 'http://192.168.1.1:5000/rootDesc.xml'; +const upnpServiceAtGatewayUrl = proxyquire( + '../../ZelBack/src/services/upnpService', + { config: { ...config, upnp: { gatewayUrl: GATEWAY_URL, nodeIp: '192.168.1.10' } }, './fluxadmPort': fluxadmPortStub }, +); + describe('upnpService tests', () => { describe('adjustFirewallForUPNP tests', () => { let originalUserConfig; @@ -336,6 +342,17 @@ describe('upnpService tests', () => { sinon.assert.notCalled(removeMappingStub); }); + it('should keep a mapping the router cannot say is to this node', async () => { + createMappingSpy.returns(true); + getMappingsStub.resolves([fluxadmMapping({ local: null })]); + + const result = await runSetup(123); + + expect(result).to.equal(true); + sinon.assert.calledOnce(getMappingsStub); + sinon.assert.notCalled(removeMappingStub); + }); + it('should neither map nor unmap the maintenance ssh port on ArcaneOS', async () => { createMappingSpy.returns(true); fluxadmPortStub.isArcane = true; @@ -554,7 +571,8 @@ describe('upnpService tests', () => { describe('removeStaleMappings tests', () => { const BUDGET_MS = 75 * 1000; - let runStub; + const LISTING_MS = 60 * 1000; + let getMappingsStub; let removeMappingStub; let clock; let table; @@ -570,33 +588,27 @@ describe('upnpService tests', () => { ...overrides, }); - // the router's error reply to an index past the last entry - const arrayIndexInvalid = () => Object.assign(new Error('Request failed with status code 500'), { response: { status: 500 } }); - const asEntry = (m) => ({ - GetGenericPortMappingEntryResponse: { - NewRemoteHost: m.public.host, - NewExternalPort: String(m.public.port), - NewProtocol: m.protocol.toUpperCase(), - NewInternalPort: String(m.private.port), - NewInternalClient: m.private.host, - NewEnabled: '1', - NewPortMappingDescription: m.description, - NewLeaseDuration: '0', - }, + const refused = () => new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping'); + // a request the router never answers: it ends when its signal aborts, as + // the library's requests do + const unanswered = ({ signal }) => new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); }); const routerHas = (mappings) => { table = mappings; }; beforeEach(() => { upnpService.mappedHere.clear(); clock = sinon.useFakeTimers(); + // the fake clock does not drive AbortSignal.timeout's own timer + sinon.stub(AbortSignal, 'timeout').callsFake((ms) => { + const controller = new AbortController(); + setTimeout(() => controller.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError')), ms); + return controller.signal; + }); sinon.stub(serviceHelper, 'delay').resolves(); table = []; - runStub = sinon.stub().callsFake(async (action, [[, index]]) => { - if (index >= table.length) throw arrayIndexInvalid(); - return asEntry(table[index]); - }); - sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves({ gateway: { run: runStub }, address: '192.168.1.10' }); - sinon.stub(natUpnp.Client.prototype, 'getMappings').rejects(new Error('the pass walks the table itself')); + getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings') + .callsFake(async (options) => table.filter((m) => !options.local || m.local === true)); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); @@ -605,25 +617,21 @@ describe('upnpService tests', () => { sinon.restore(); }); - it('should walk the router\'s table one index at a time, up to the error reply that ends it', async () => { - routerHas([mapping(31000, 'Flux_App_gone'), mapping(31001, 'Flux_App_sibling', { private: { host: '192.168.1.11', port: 31001 } })]); + it('should list only the router\'s mappings to this node, and remove the stale one', async () => { + routerHas([mapping(31000, 'Flux_App_gone'), mapping(31001, 'Flux_App_sibling', { local: false, private: { host: '192.168.1.11', port: 31001 } })]); const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); - expect(runStub.getCalls().map((call) => [call.args[0], call.args[1][0][1]])).to.deep.equal([ - ['GetGenericPortMappingEntry', 0], - ['GetGenericPortMappingEntry', 1], - ['GetGenericPortMappingEntry', 2], - ]); + sinon.assert.calledOnceWithExactly(getMappingsStub, { local: true, signal: sinon.match.instanceOf(AbortSignal) }); expect(removed).to.equal(1); - sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP' }); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP', signal: sinon.match.instanceOf(AbortSignal) }); }); - it('should take an error reply at the first index for an empty table', async () => { + it('should remove nothing from an empty table', async () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); expect(removed).to.equal(0); - sinon.assert.calledOnce(runStub); + sinon.assert.calledOnce(getMappingsStub); sinon.assert.notCalled(removeMappingStub); }); @@ -633,7 +641,7 @@ describe('upnpService tests', () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); expect(removed).to.equal(1); - sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'UDP' }); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'UDP', signal: sinon.match.instanceOf(AbortSignal) }); }); it('should keep every mapping on a port something holds, whatever its description', async () => { @@ -660,9 +668,8 @@ describe('upnpService tests', () => { expect(removeMappingStub.getCalls().map((call) => call.args[0].public.port)).to.deep.equal([16127, 16128, 16126, 16129]); }); - it('should never touch a mapping to another address, the operator\'s manual entries or anything it did not make', async () => { + it('should never touch the operator\'s manual entries or anything it did not make', async () => { routerHas([ - mapping(31000, 'Flux_App_sibling', { local: false, private: { host: '192.168.1.11', port: 31000 } }), mapping(31001, 'Flux_manual_entry'), mapping(22, 'ssh'), mapping(31002, undefined), @@ -687,7 +694,7 @@ describe('upnpService tests', () => { it('should go on past a removal the router refuses', async () => { routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_Prelaunch_App_31001')]); - removeMappingStub.onFirstCall().rejects(new Error('ActionFailed')); + removeMappingStub.onFirstCall().rejects(refused()); const warnSpy = sinon.spy(log, 'warn'); const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); @@ -697,37 +704,24 @@ describe('upnpService tests', () => { sinon.assert.calledOnce(warnSpy); }); - it('should reject, and remove nothing, when the router does not reply to the listing at all', async () => { - routerHas([mapping(31000, 'Flux_App_gone'), mapping(31001, 'Flux_App_gone2')]); - runStub.onSecondCall().rejects(Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' })); + it('should reject, and remove nothing, when the listing fails', async () => { + getMappingsStub.rejects(Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' })); let error; await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS).catch((err) => { error = err; }); expect(error.message).to.equal('socket hang up'); - sinon.assert.calledTwice(runStub); - sinon.assert.notCalled(removeMappingStub); - }); - - it('should reject on a reply that is not a mapping entry', async () => { - runStub.resolves({ SomethingElse: {} }); - - let error; - await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS).catch((err) => { error = err; }); - - expect(error.message).to.equal('listing the mappings: incorrect response'); sinon.assert.notCalled(removeMappingStub); }); it('should ask what is held only after the router has been listed', async () => { routerHas([mapping(31000, 'Flux_App_a')]); - let requestsBeforeHeld; - const held = sinon.stub().callsFake(async () => { requestsBeforeHeld = runStub.callCount; return [16127]; }); + let listingsBeforeHeld; + const held = sinon.stub().callsFake(async () => { listingsBeforeHeld = getMappingsStub.callCount; return [16127]; }); await upnpService.removeStaleMappings(held, BUDGET_MS); - expect(requestsBeforeHeld).to.equal(2); - expect(runStub.callCount).to.equal(2); + expect(listingsBeforeHeld).to.equal(1); }); it('should remove nothing when what is held cannot be read', async () => { @@ -775,7 +769,7 @@ describe('upnpService tests', () => { it('should remove a port mapUpnpPort mapped once removeMapUpnpPort has given it up, even if the router refused', async () => { sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); await upnpService.mapUpnpPort(31000, 'Flux_App_gone'); - removeMappingStub.onFirstCall().rejects(new Error('ActionFailed')); + removeMappingStub.onFirstCall().rejects(refused()); await upnpService.removeMapUpnpPort(31000); removeMappingStub.reset(); removeMappingStub.resolves(); @@ -787,53 +781,42 @@ describe('upnpService tests', () => { sinon.assert.calledOnce(removeMappingStub); }); - it('should send the router nothing more once a listing request goes unanswered', async () => { - routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b')]); - runStub.onSecondCall().returns(new Promise(() => {})); + it('should give the listing its 60s of the 75s slot, and remove nothing when it does not finish', async () => { + routerHas([mapping(31000, 'Flux_App_a')]); + getMappingsStub.callsFake(unanswered); let error; const passing = upnpService.removeStaleMappings(async () => [], BUDGET_MS).catch((err) => { error = err; }); - await clock.tickAsync(BUDGET_MS); + await clock.tickAsync(LISTING_MS - 1); + expect(error).to.equal(undefined); + await clock.tickAsync(1); await passing; - expect(error.message).to.match(/listing the mappings: the router did not answer within 15s/); - sinon.assert.calledTwice(runStub); + expect(error.name).to.equal('TimeoutError'); sinon.assert.notCalled(removeMappingStub); }); - it('should stop walking a slow router\'s table before the slot ends, and remove from what it listed', async () => { - // 100 entries at 1s each would take 100s; the listing has 60s of the 75s slot - routerHas(Array.from({ length: 100 }, (_, i) => mapping(31000 + i, `Flux_App_${i}`))); - runStub.callsFake(async (action, args) => { - await new Promise((resolve) => { setTimeout(resolve, 1000); }); - const [[, index]] = args; - if (index >= table.length) throw arrayIndexInvalid(); - return asEntry(table[index]); - }); + it('should step past a refused removal but end the pass on one the router does not answer', async () => { + routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b'), mapping(31002, 'Flux_App_c')]); + removeMappingStub.onFirstCall().rejects(refused()); + removeMappingStub.onSecondCall().callsFake(unanswered); const passing = upnpService.removeStaleMappings(async () => [], BUDGET_MS); await clock.tickAsync(BUDGET_MS); const removed = await passing; - // no request goes out after the listing's 60s: at most the one in flight at that moment - expect(runStub.callCount).to.be.within(60, 61); - expect(removed).to.equal(runStub.callCount); - const requestsBySlotEnd = runStub.callCount; - await clock.tickAsync(10 * 60 * 1000); - expect(runStub.callCount).to.equal(requestsBySlotEnd); + expect(removed).to.equal(0); + sinon.assert.calledTwice(removeMappingStub); }); - it('should step past a refused removal but end the pass on one the router does not answer', async () => { - routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b'), mapping(31002, 'Flux_App_c')]); - removeMappingStub.onFirstCall().rejects(new Error('ActionFailed')); - removeMappingStub.onSecondCall().returns(new Promise(() => {})); + it('should end the pass on a removal that fails without a UPnP error', async () => { + routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b')]); + removeMappingStub.onFirstCall().rejects(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })); - const passing = upnpService.removeStaleMappings(async () => [], BUDGET_MS); - await clock.tickAsync(15 * 1000); - const removed = await passing; + const removed = await upnpService.removeStaleMappings(async () => [], BUDGET_MS); expect(removed).to.equal(0); - sinon.assert.calledTwice(removeMappingStub); + sinon.assert.calledOnce(removeMappingStub); }); it('should stop removing once its budget is spent, and report what it did', async () => { @@ -1318,23 +1301,32 @@ describe('upnpService tests', () => { }); }); - it('should show a proper message if all data is valid', async () => { - getGatewaySpy.resolves('10.1.1.1'); + it('should answer with the gateway\'s description and services and this node\'s address', async () => { + const device = new natUpnp.Device(GATEWAY_URL); + getGatewaySpy.resolves(new natUpnp.UpnpInfo(device, '192.168.1.10')); verifyPrivilegeStub.resolves(true); - const req = { - params: { - port: '1234', - }, - query: { - test: 'test', - }, - }; const res = generateResponse(); - await upnpService.getGatewayApi(req, res); + await upnpService.getGatewayApi({ params: {}, query: {} }, res); sinon.assert.calledOnce(getGatewaySpy); - sinon.assert.calledOnceWithExactly(res.json, { status: 'success', data: '10.1.1.1' }); + sinon.assert.calledOnceWithExactly(res.json, { + status: 'success', + data: { gateway: { description: GATEWAY_URL, services: device.services }, address: '192.168.1.10' }, + }); + }); + + it('should reach the router at config.upnp.gatewayUrl as config.upnp.nodeIp, without discovering it', async () => { + getGatewaySpy.restore(); + verifyPrivilegeStub.resolves(true); + const res = generateResponse(); + + await upnpServiceAtGatewayUrl.getGatewayApi({ params: {}, query: {} }, res); + + sinon.assert.calledOnceWithExactly(res.json, { + status: 'success', + data: { gateway: { description: GATEWAY_URL, services: new natUpnp.Device(GATEWAY_URL).services }, address: '192.168.1.10' }, + }); }); }); }); From aff36fd19152690d3dbe64349659ac92f00eab3c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 14:20:45 +0100 Subject: [PATCH 003/369] test(apiServer): handleSigterm tests stub the shutdown-reason write writeShutdownReason writes to the local database whenever a connection is open. A test file that ran first and left one open sent the write to a real MongoDB under the pause test's fake setTimeout, so the shutdown never reached the pause within the time the test advanced, and the next test joined the shutdown still in flight. Co-Authored-By: Claude Opus 5.5 --- tests/unit/apiServer.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/apiServer.test.js b/tests/unit/apiServer.test.js index a55be53f82..d86b82512e 100644 --- a/tests/unit/apiServer.test.js +++ b/tests/unit/apiServer.test.js @@ -12,6 +12,7 @@ const globalState = require('../../ZelBack/src/services/utils/globalState'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const fluxCommunicationMessagesSender = require('../../ZelBack/src/services/fluxCommunicationMessagesSender'); const messageStore = require('../../ZelBack/src/services/appMessaging/messageStore'); +const { AppSyncOrchestrator } = require('../../ZelBack/src/services/appMessaging/appSyncOrchestrator'); const { PM2_KILL_TIMEOUT_MS } = require('../../ZelBack/src/services/fluxService'); const serviceManager = require('../../ZelBack/src/services/serviceManager'); const systemdNotify = require('../../ZelBack/src/services/utils/systemdNotify'); @@ -613,6 +614,8 @@ describe('handleSigterm drains syncthing folders before it exits', () => { drainStub = sinon.stub(syncthingService, 'drainFoldersToPeers').resolves([]); sinon.stub(syncthingFolderWrites, 'pauseAllFolders').resolves({ paused: [], failed: [] }); sinon.stub(verifyPool, 'stop'); + // A database another test file left connected would take the write. + sinon.stub(AppSyncOrchestrator, 'writeShutdownReason').resolves(); exitStub = sinon.stub(process, 'exit'); // The flag is one-way and module-wide; set for real it would refuse every // container start in the rest of this mocha process. From 55aa1e958aad8f6504df4b311adc0bebd674789d Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 14:56:31 +0100 Subject: [PATCH 004/369] perf(upnp): keep the discovered router, and look up one mapping instead of walking the table The client keeps the router it discovers between calls (nat-upnp 2.0.1 finds it again when it stops answering where it was), so a mapping no longer costs an SSDP search. verifyUPNPsupport looks up the test mapping it has just made, and fails when the router does not hold it. reconcileFluxadmMapping looks up the maintenance ssh port's TCP mapping instead of listing the table. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 17 ++-- package.json | 2 +- tests/unit/upnpService.test.js | 131 +++++++++++++++++++++------- 3 files changed, 109 insertions(+), 41 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 5893cba80d..9e57dd494f 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -12,9 +12,11 @@ const { Privilege, authOf } = require('./utils/privileges'); // config.upnp.gatewayUrl names the router's device description, used in place // of discovering the router; the node's address is then config.upnp.nodeIp. +// A discovered router is kept between calls, and found again when it stops +// answering where it was. const client = config.upnp.gatewayUrl ? new natUpnp.Client({ url: config.upnp.gatewayUrl, localAddress: config.upnp.nodeIp || '127.0.0.1' }) - : new natUpnp.Client(); + : new natUpnp.Client({ cacheGateway: true }); const FLUXADM_MAPPING_DESCRIPTION = 'Flux_Fluxadm_SSH'; @@ -145,12 +147,13 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { return false; } try { - await client.getMappings(); + const mapping = await client.getMapping({ public: +apiport + 3, protocol: 'TCP' }); + if (!mapping) throw new Error(`the router holds no mapping for port ${+apiport + 3}`); await serviceHelper.delay(500); } catch (error) { log.error(error); - log.error('VerifyUPNPsupport - Failed get Mappings'); + log.error('VerifyUPNPsupport - Failed get Mapping'); upnpMachine = false; return false; } @@ -199,10 +202,10 @@ async function reconcileFluxadmMapping(apiport) { }); return; } - const mappings = await client.getMappings(); - const ours = mappings.some((mapping) => mapping.local - && mapping.public.port === port - && mapping.description === FLUXADM_MAPPING_DESCRIPTION); + const mapping = await client.getMapping({ public: port, protocol: 'TCP' }); + const ours = mapping !== null + && mapping.local === true + && mapping.description === FLUXADM_MAPPING_DESCRIPTION; if (ours) { await client.removeMapping({ public: port, protocol: 'TCP' }); log.info(`fluxadm access - UPnP mapping for port ${port} removed`); diff --git a/package.json b/package.json index 934e8ff1bf..69b2b4a90f 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "@aws-sdk/client-ecr": "^3.922.0", "@azure/identity": "^4.13.0", "@isaacs/ttlcache": "~1.4.1", - "@runonflux/nat-upnp": "~2.0.0", + "@runonflux/nat-upnp": "~2.0.1", "apicache": "~1.6.3", "archiver": "~7.0.1", "axios": "~1.13.6", diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index d889cbf643..a65332f130 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -44,6 +44,19 @@ const upnpService = proxyquire( { config, './fluxadmPort': fluxadmPortStub }, ); +// the options each nat-upnp client was constructed with +const clientOptions = []; +class RecordingClient extends natUpnp.Client { + constructor(options) { + super(options); + clientOptions.push(options); + } +} +proxyquire( + '../../ZelBack/src/services/upnpService', + { config, './fluxadmPort': fluxadmPortStub, '@runonflux/nat-upnp': { ...natUpnp, Client: RecordingClient } }, +); + const GATEWAY_URL = 'http://192.168.1.1:5000/rootDesc.xml'; const upnpServiceAtGatewayUrl = proxyquire( '../../ZelBack/src/services/upnpService', @@ -51,6 +64,12 @@ const upnpServiceAtGatewayUrl = proxyquire( ); describe('upnpService tests', () => { + describe('client tests', () => { + it('should keep the router it discovers between calls', () => { + expect(clientOptions).to.deep.equal([{ cacheGateway: true }]); + }); + }); + describe('adjustFirewallForUPNP tests', () => { let originalUserConfig; @@ -106,6 +125,16 @@ describe('upnpService tests', () => { describe('verifyUPNPsupport tests', () => { let logSpy; + // what the router answers for the test mapping verifyUPNPsupport makes + const testMapping = { + public: { host: '', port: 16130 }, + private: { host: '192.168.1.10', port: 16130 }, + protocol: 'tcp', + enabled: true, + description: 'Flux_UPNP_Mapping_Test', + ttl: 0, + local: true, + }; beforeEach(() => { logSpy = sinon.spy(log, 'error'); @@ -119,7 +148,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -139,7 +168,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); sinon.stub(natUpnp.Client.prototype, 'createMapping').callsFake(async () => { heldAt.create = upnpService.mappedHere.has(16130); }); - sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').callsFake(async () => { heldAt.remove = upnpService.mappedHere.has(16130); }); const clock = sinon.useFakeTimers(); @@ -157,7 +186,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').throws(); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -176,7 +205,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').throws(); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -195,7 +224,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').throws(); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -210,11 +239,44 @@ describe('upnpService tests', () => { sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping'); }); - it('should log a proper error if getMappings throws', async () => { + it('should look up the test mapping it made, by its port, and nothing else', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(true); + const getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); + const getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(true); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(2_500); + + expect(await promise).to.equal(true); + sinon.assert.calledOnceWithExactly(getMappingStub, { public: 16130, protocol: 'TCP' }); + sinon.assert.notCalled(getMappingsStub); + }); + + it('should fail when the router holds no mapping for the test port it just made', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(true); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(2_500); + + expect(await promise).to.equal(false); + sinon.assert.calledTwice(logSpy); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mapping'); + }); + + it('should log a proper error if getMapping throws', async () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').throws(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').throws(); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -226,14 +288,14 @@ describe('upnpService tests', () => { expect(result).to.equal(false); sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mappings'); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mapping'); }); it('should log a proper error if removeMapping throws', async () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').throws(); const clock = sinon.useFakeTimers(); @@ -252,7 +314,7 @@ describe('upnpService tests', () => { describe('setupUPNP tests', () => { let logSpy; let createMappingSpy; - let getMappingsStub; + let getMappingStub; let removeMappingStub; let unitStateStub; @@ -265,7 +327,7 @@ describe('upnpService tests', () => { .withArgs('systemctl', sinon.match({ params: ['is-enabled', 'fluxadm-sshd.socket'] })); sshdSocket('disabled\n'); createMappingSpy = sinon.stub(natUpnp.Client.prototype, 'createMapping'); - getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); @@ -307,63 +369,66 @@ describe('upnpService tests', () => { it('should remove its own maintenance ssh mapping while its socket is not enabled', async () => { createMappingSpy.returns(true); - getMappingsStub.resolves([fluxadmMapping()]); + getMappingStub.resolves(fluxadmMapping()); const result = await runSetup(123); expect(result).to.equal(true); sinon.assert.callCount(createMappingSpy, 4); + sinon.assert.calledOnceWithExactly(getMappingStub, { public: 118, protocol: 'TCP' }); sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); }); - it('should remove its own maintenance ssh mapping once its socket is gone, or systemctl cannot answer', async () => { + it('should remove nothing when the router holds no mapping of the maintenance ssh port', async () => { createMappingSpy.returns(true); - getMappingsStub.resolves([fluxadmMapping()]); - sshdSocket('', new Error('Failed to get unit file state for fluxadm-sshd.socket: No such file or directory')); const result = await runSetup(123); expect(result).to.equal(true); - sinon.assert.callCount(createMappingSpy, 4); - sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); + sinon.assert.calledOnce(getMappingStub); + sinon.assert.notCalled(removeMappingStub); }); - it('should keep a mapping of the same port that it did not make', async () => { + it('should remove its own maintenance ssh mapping once its socket is gone, or systemctl cannot answer', async () => { createMappingSpy.returns(true); - getMappingsStub.resolves([ - fluxadmMapping({ description: 'node owner ssh' }), - fluxadmMapping({ local: false }), - ]); + getMappingStub.resolves(fluxadmMapping()); + sshdSocket('', new Error('Failed to get unit file state for fluxadm-sshd.socket: No such file or directory')); const result = await runSetup(123); expect(result).to.equal(true); - sinon.assert.calledOnce(getMappingsStub); - sinon.assert.notCalled(removeMappingStub); + sinon.assert.callCount(createMappingSpy, 4); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); }); - it('should keep a mapping the router cannot say is to this node', async () => { - createMappingSpy.returns(true); - getMappingsStub.resolves([fluxadmMapping({ local: null })]); + [ + ['made under another description', { description: 'node owner ssh' }], + ['to another machine', { local: false }], + ['the router cannot say is to this node', { local: null }], + ].forEach(([what, overrides]) => { + it(`should keep a mapping of the same port ${what}`, async () => { + createMappingSpy.returns(true); + getMappingStub.resolves(fluxadmMapping(overrides)); - const result = await runSetup(123); + const result = await runSetup(123); - expect(result).to.equal(true); - sinon.assert.calledOnce(getMappingsStub); - sinon.assert.notCalled(removeMappingStub); + expect(result).to.equal(true); + sinon.assert.calledOnce(getMappingStub); + sinon.assert.notCalled(removeMappingStub); + }); }); it('should neither map nor unmap the maintenance ssh port on ArcaneOS', async () => { createMappingSpy.returns(true); fluxadmPortStub.isArcane = true; sshdSocket('enabled\n'); - getMappingsStub.resolves([fluxadmMapping()]); + getMappingStub.resolves(fluxadmMapping()); const result = await runSetup(123); expect(result).to.equal(true); sinon.assert.callCount(createMappingSpy, 4); - sinon.assert.notCalled(getMappingsStub); + sinon.assert.notCalled(getMappingStub); sinon.assert.notCalled(removeMappingStub); }); From 4ac264a18ec6de4b9b294e3ae2d6841b56cce1fb Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 17:17:16 +0100 Subject: [PATCH 005/369] test(harness): the stub gateway answers GetSpecificPortMappingEntry verifyUPNPsupport confirms its test mapping with a single lookup, so a node behind the stub gateway (2226, 2227) verifies UPnP only if the gateway answers one: the entry for that external port and protocol, or 714 when it holds none. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 4a3776424e..3c0b94abbf 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -715,7 +715,7 @@ app.get('/repos/:owner/:repo', (req, res) => { // fails on it, and the node runs without UPnP as it does on a network with none. // // A node behind a gateway is served one: a WANIPConnection service whose control URL answers -// the four actions upnpService sends - the external address, and adding, listing and +// the actions upnpService sends - the external address, and adding, looking up, listing and // deleting port mappings - from that node's own mapping table. const UPNP_SERVICE = 'urn:schemas-upnp-org:service:WANIPConnection:1'; @@ -790,6 +790,21 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { else soapReply(res, action, {}); return; } + case 'GetSpecificPortMappingEntry': { + const entry = gateway.mappings.find(sameEntry); + if (!entry) { + soapFault(res, 714, 'NoSuchEntryInArray'); + return; + } + soapReply(res, action, { + NewInternalPort: entry.internalPort, + NewInternalClient: entry.internalClient, + NewEnabled: 1, + NewPortMappingDescription: entry.description, + NewLeaseDuration: entry.leaseDuration, + }); + return; + } case 'GetGenericPortMappingEntry': { const entry = gateway.mappings[Number(arg('NewPortMappingIndex'))]; if (!entry) { From 028bf23885498f8784381f1681dfe011b8ea7e75 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 17:31:49 +0100 Subject: [PATCH 006/369] fix(upnp): a full router table is scored as the router's, and a node still verifies on one mapUpnpPort answers { mapped, code, reason, routerWide } in place of a boolean. routerWide is true only for a refusal that is the router's own - a full table (728) or no answer - and never for a port another machine holds (718). Each such refusal counts in routerRefusals(), which only goes up. - One mapping step per protocol, shared by the verify, the core ports and the app ports: a refusal is no failure when the router already maps the port to this node, enabled, port for port. A router that does not answer is not asked again. - The availability check reads routerRefusals() once per pass: any rise since the last pass - an install's, a restore's or its own test mapping's - counts 4 toward the router DOS, however many refusals there were. A pass whose port test works ends this check's DOS only when the router refused nothing since the last. - A port another machine holds stays per-app: portManager removes the app after its sustained failure window, as before. - removeMapUpnpPort looks up each protocol and removes only what points at this node; 714 counts as removed, and a refusal of TCP still removes UDP. The availability check removes what the router kept of a test mapping it refused. - An install, a prelaunch test and the core mapping report the router's reason. - verifyUPNPsupport on a full table removes the test mappings earlier runs left - a port test's, a prelaunch test's, its own - and maps once more. Until UPnP is verified the stale mapping cleanup never runs, so nothing else frees an entry. The listing and removal share the cleanup's loop. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appInstaller.js | 8 +- .../appMonitoring/availabilityChecker.js | 33 +- .../src/services/appNetwork/portManager.js | 8 +- ZelBack/src/services/serviceManager.js | 2 +- ZelBack/src/services/upnpService.js | 347 ++++++++++------ tests/unit/appInstaller.test.js | 38 +- tests/unit/availabilityChecker.test.js | 271 +++++++------ tests/unit/fluxNetworkHelper.test.js | 4 +- tests/unit/portManager.test.js | 38 +- tests/unit/upnpService.test.js | 369 +++++++++++++++--- 10 files changed, 787 insertions(+), 331 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 6da5b03dab..4d3003a5ba 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -124,7 +124,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re for (const port of appSpecifications.ports) { // eslint-disable-next-line no-await-in-loop const portResponse = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${appName}`); - if (portResponse === true) { + if (portResponse.mapped) { const portStatus = { status: `Port ${port} mapped OK`, }; @@ -134,7 +134,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (res.flush) res.flush(); } } else { - throw new Error(`Error: Port ${port} FAILed to map.`); + throw new Error(`Error: Port ${port} FAILed to map. ${portResponse.reason}`); } } } @@ -144,7 +144,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (isUPNP) { log.info('Custom port specified, mapping ports'); const portResponse = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(appSpecifications.port), `Flux_App_${appName}`); - if (portResponse === true) { + if (portResponse.mapped) { const portStatus = { status: `Port ${appSpecifications.port} mapped OK`, }; @@ -154,7 +154,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (res.flush) res.flush(); } } else { - throw new Error(`Error: Port ${appSpecifications.port} FAILed to map.`); + throw new Error(`Error: Port ${appSpecifications.port} FAILed to map. ${portResponse.reason}`); } } } diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index ae688e351a..de3fb93891 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -97,6 +97,22 @@ function releaseOwnDos(dosState) { dosState.dosStateValue = 0; } +/** + * Whether the router has refused a mapping for a reason of its own - a full + * table, no answer - since this check last asked: any mapping, an install's, a + * restore's or this check's own. Read once per pass, so a pass counts once + * however many it refused. + * @param {object} dosState + * @returns {boolean} + */ +function routerRefusedSinceLastPass(dosState) { + const refusals = upnpService.routerRefusals(); + const refused = refusals > dosState.routerRefusalsSeen; + // eslint-disable-next-line no-param-reassign + dosState.routerRefusalsSeen = refusals; + return refused; +} + /** * Check my apps availability by testing ports from external nodes * @param {Function} installedAppsFn - Function to get installed apps @@ -110,6 +126,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const { timeoutsMs: timeouts, thresholds } = config.fluxapps.appsPortCheck; const isUpnp = upnpService.isUPNP(); + let routerRefused = false; const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(); const setNextPort = () => { @@ -251,15 +268,14 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isUpnp) { const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App'); - if (!upnpMapResult) { - if (dosState.lastUPNPMapFailed) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); - // eslint-disable-next-line no-param-reassign - dosState.lastUPNPMapFailed = true; - log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} failed to create UPnP mapping`); + routerRefused = routerRefusedSinceLastPass(dosState); + if (routerRefused) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); + if (!upnpMapResult.mapped) { + log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} failed to create UPnP mapping: ${upnpMapResult.reason}`); setNextPort(); + // the router may have kept the TCP half await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, - skipUpnp: true, skipHttpServer: true, }); const upnpDelay = dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error; @@ -267,8 +283,6 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } - // eslint-disable-next-line no-param-reassign - dosState.lastUPNPMapFailed = false; } const listening = new Promise((resolve, reject) => { @@ -396,7 +410,8 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin dosState.nextTestingPort = null; // eslint-disable-next-line no-param-reassign dosState.originalPortFailed = null; - releaseOwnDos(dosState); + // a router refusing others' mappings is not cleared by one that took this test's + if (!routerRefused) releaseOwnDos(dosState); waitMs = timeouts.default; } diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 479ac7c33a..42fe960f1a 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -298,7 +298,7 @@ async function restoreAppsPortsSupport() { // eslint-disable-next-line no-restricted-syntax for (const port of application.ports) { // eslint-disable-next-line no-await-in-loop - let upnpOk = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`); + let upnpOk = (await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`)).mapped; if (!upnpOk) { if (!retryDelayElapsed) { // eslint-disable-next-line no-await-in-loop @@ -306,7 +306,7 @@ async function restoreAppsPortsSupport() { retryDelayElapsed = true; } // eslint-disable-next-line no-await-in-loop - upnpOk = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`); + upnpOk = (await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`)).mapped; } if (!upnpOk) { failedPort = port; @@ -874,8 +874,8 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { if (isUPNP) { // eslint-disable-next-line no-await-in-loop const upnpMapResult = await upnpService.mapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`); - if (!upnpMapResult) { - throw new Error('Failed to create map UPNP port'); + if (!upnpMapResult.mapped) { + throw new Error(`Failed to create map UPNP port: ${upnpMapResult.reason}`); } } const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(portTestToken); diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index ab1acdc08b..174a92a9cb 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -99,7 +99,7 @@ const dosState = { testingPort: null, nextTestingPort: null, originalPortFailed: null, - lastUPNPMapFailed: false, + routerRefusalsSeen: 0, }; const portsNotWorking = new Set(); const appsStorageViolations = []; diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 9e57dd494f..0bed2b8adc 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -52,6 +52,88 @@ const CLEANUP_REMOVAL_RESERVE_MS = 15 * 1000; const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); +// The router's answers this code acts on: a full table, and no such entry. +const NO_PORT_MAPS_AVAILABLE = 728; +const NO_SUCH_ENTRY = 714; + +// What a test leaves on the router while it runs and removes before it ends. +const TEST_MAPPING_DESCRIPTIONS = new Set(['Flux_UPNP_Mapping_Test', 'Flux_Test_App']); +const TEST_MAPPING_PREFIX = 'Flux_Prelaunch_App_'; + +const MAPPED = Object.freeze({ + mapped: true, code: null, reason: null, routerWide: false, +}); + +let routerRefusalCount = 0; + +/** + * How many mappings the router has refused since FluxOS started for a reason + * that is the router's own and not the port's: a full table, or no answer. A + * port another machine holds (718) and every other refusal are the port's. + * Only ever goes up. + * @returns {number} + */ +function routerRefusals() { + return routerRefusalCount; +} + +/** + * Whether the router maps `port` to this node, port for port, enabled. + * @param {object|null} mapping the router's entry for the port, as client.getMapping reads it + * @param {number} port + * @returns {boolean} + */ +function mapsHere(mapping, port) { + return mapping !== null && mapping.local === true && mapping.enabled && mapping.private.port === port; +} + +/** + * Maps one protocol of a port to this node, port for port, with no lease + * (some routers force a short one instead; the mappings are refreshed every + * restore). A refusal is no failure when the router already maps the port + * here: some routers refuse to add an entry they hold, and a full table + * refuses everything, the refresh of a working mapping included. A router + * that does not answer is not asked again. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @param {string} description + * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} + * `code` is the router's UPnP error code, null when it gave none; `routerWide` + * whether the refusal counts in routerRefusals. + */ +async function mapProtocol(port, protocol, description) { + if (!Number.isInteger(port) || port < 1 || port > 65535) { + return { + mapped: false, code: null, reason: `${protocol} ${port} not mapped: not a port`, routerWide: false, + }; + } + try { + await client.createMapping({ + public: port, + private: port, + ttl: 0, + protocol, + description, + }); + return MAPPED; + } catch (error) { + const answered = error instanceof natUpnp.UpnpError; + if (answered) { + const held = await client.getMapping({ public: port, protocol }).catch(() => null); + if (mapsHere(held, port)) { + log.info(`UPnP - ${protocol} ${port} refused (${error.message}), but the router already maps it to this node`); + return MAPPED; + } + } + const code = answered ? error.code : null; + const routerWide = !answered || code === NO_PORT_MAPS_AVAILABLE; + if (routerWide) routerRefusalCount += 1; + return { + mapped: false, code, reason: `${protocol} ${port} not mapped: ${error.message}`, routerWide, + }; + } +} + let upnpMachine = false; /** @@ -131,21 +213,24 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { // the remove would fail the verify, and a node with a custom api port or a // router IP exits on that. A test that fails leaves it held. mappedHere.add(+apiport + 3); - try { - await client.createMapping({ - public: +apiport + 3, - private: +apiport + 3, - ttl: 0, - description: 'Flux_UPNP_Mapping_Test', - }); - - await serviceHelper.delay(500); - } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed Create Mapping'); + let testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test'); + // a full table: until UPnP is verified the stale mapping cleanup never runs, + // so the test mappings earlier runs left are removed here to make room + if (testMapping.code === NO_PORT_MAPS_AVAILABLE) { + let removed = 0; + try { + removed = await removeLeftoverTestMappings(); + } catch (error) { + log.warn(`VerifyUPNPsupport - left-over test mappings not removed: ${error.message}`); + } + if (removed > 0) testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test'); + } + if (!testMapping.mapped) { + log.error(`VerifyUPNPsupport - Failed Create Mapping: ${testMapping.reason}`); upnpMachine = false; return false; } + await serviceHelper.delay(500); try { const mapping = await client.getMapping({ public: +apiport + 3, protocol: 'TCP' }); if (!mapping) throw new Error(`the router holds no mapping for port ${+apiport + 3}`); @@ -222,41 +307,23 @@ async function reconcileFluxadmMapping(apiport) { */ async function setupUPNP(apiport = config.server.apiport) { try { - await client.createMapping({ - public: +apiport, - private: +apiport, - ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes - description: 'Flux_Backend_API', - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: +apiport + 1, - private: +apiport + 1, - ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes - description: 'Flux_Backend_API_SSL', - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: +apiport - 1, - private: +apiport - 1, - ttl: 0, - description: 'Flux_Home_UI', - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: +apiport + 2, - private: +apiport + 2, - ttl: 0, - description: 'Flux_Syncthing', - }); - - await serviceHelper.delay(500); + const corePortMappings = [ + [+apiport, 'Flux_Backend_API'], + [+apiport + 1, 'Flux_Backend_API_SSL'], + [+apiport - 1, 'Flux_Home_UI'], + [+apiport + 2, 'Flux_Syncthing'], + ]; + // eslint-disable-next-line no-restricted-syntax + for (const [port, description] of corePortMappings) { + // eslint-disable-next-line no-await-in-loop + const result = await mapProtocol(port, 'TCP', description); + if (!result.mapped) { + log.error(`setupUPNP - ${result.reason}`); + return false; + } + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(500); + } await reconcileFluxadmMapping(apiport); @@ -268,71 +335,81 @@ async function setupUPNP(apiport = config.server.apiport) { } /** - * To create mappings for UPnP (Universal Plug and Play) port. + * Maps a port to this node over TCP and then UDP. + * + * On a failure the port is no longer held here: a TCP half the router kept is + * the caller's to remove with removeMapUpnpPort, or the stale mapping + * cleanup's. * @param {number} port Port number. * @param {string} description Port description. - * @returns {Promise} True if port mappings can be created for both TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) protocols. Otherwise false. + * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} + * the first refusal, as mapProtocol gives it; mapped when both protocols are */ async function mapUpnpPort(port, description) { // held from before the router has it: the cleanup pass may list it the // moment it is there mappedHere.add(+port); - try { - await client.createMapping({ - public: port, - private: port, - ttl: 0, - protocol: 'TCP', - description, - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: port, - private: port, - ttl: 0, - protocol: 'UDP', - description, - }); - + let result = await mapProtocol(+port, 'TCP', description); + if (result.mapped) { await serviceHelper.delay(500); + result = await mapProtocol(+port, 'UDP', description); + } + if (!result.mapped) { + log.error(`UPnP - ${result.reason}`); + mappedHere.delete(+port); + return result; + } + await serviceHelper.delay(500); + return result; +} +/** + * Removes the router's mapping of one protocol of a port when it points at + * this node. A router removes a mapping by its external port alone, whoever it + * points at, so the entry is looked up first: one pointing at another machine + * - another node behind the same router holding that port - is left, and one + * the router does not hold is gone already. An entry that cannot be looked up + * is removed. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @returns {Promise} whether no mapping of it to this node is left + */ +async function removeProtocol(port, protocol) { + let held; + try { + held = await client.getMapping({ public: port, protocol }); + } catch (error) { + log.warn(`UPnP - ${protocol} ${port} not looked up, removing it: ${error.message}`); + } + if (held === null) return true; + if (held && held.local === false) { + log.info(`UPnP - ${protocol} ${port} left mapped: the router maps it to ${held.private.host}`); + return true; + } + try { + await client.removeMapping({ public: port, protocol }); return true; } catch (error) { - log.error(error); + if (error instanceof natUpnp.UpnpError && error.code === NO_SUCH_ENTRY) return true; + log.error(`UPnP - ${protocol} ${port} not unmapped: ${error.message}`); return false; } } /** - * To remove TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) port mappings from UPnP (Universal Plug and Play) port. + * Removes the TCP and the UDP mapping of a port to this node, each on its own: + * a refusal of one leaves the other to go. * @param {number} port Port number. - * @returns {Promise} True if port mappings have been removed for both TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) protocols. Otherwise false. + * @returns {Promise} whether no mapping of the port to this node is left */ async function removeMapUpnpPort(port) { // given up even if the router refuses: what is left is the cleanup pass's mappedHere.delete(+port); - try { - await client.removeMapping({ - public: port, - protocol: 'TCP', - }); - - await serviceHelper.delay(500); - - await client.removeMapping({ - public: port, - protocol: 'UDP', - }); - - await serviceHelper.delay(500); - - return true; - } catch (error) { - log.error(error); - return false; - } + const tcpRemoved = await removeProtocol(+port, 'TCP'); + await serviceHelper.delay(500); + const udpRemoved = await removeProtocol(+port, 'UDP'); + await serviceHelper.delay(500); + return tcpRemoved && udpRemoved; } /** @@ -347,6 +424,16 @@ function isOwnedMappingDescription(description) { || OWNED_MAPPING_PREFIXES.some((prefix) => description.startsWith(prefix)); } +/** + * Whether a mapping's description is a test's. + * @param {string} description + * @returns {boolean} + */ +function isTestMappingDescription(description) { + return typeof description === 'string' + && (TEST_MAPPING_DESCRIPTIONS.has(description) || description.startsWith(TEST_MAPPING_PREFIX)); +} + /** * The ports FluxOS itself maps for an api port: the home UI, the api and its * SSL port, syncthing and the maintenance sshd. @@ -369,21 +456,11 @@ function msUntilCleanupSlot(apiport, now = Date.now()) { } /** - * Removes the router's mappings to this node that this code made and nothing - * here holds any more: an app removed while the router was unreachable, a - * FluxOS that died between mapping and unmapping, a node reinstalled on the - * same address without the apps it ran, an api port that moved. They are made - * with an indefinite lease (or the router's longest, a week under IGDv2), so - * nothing else ever takes them, and home routers hold only so many. + * Removes the router's mappings to this node that `select` picks from them. * - * One listing. Only mappings to this node's own address are looked at - - * several nodes behind one router is an ordinary setup - and only those with a - * description this code gives, on a port nothing here holds and mapUpnpPort has - * not mapped since FluxOS started. - * - * What is held is asked for only after the router has been listed: an app's - * record is written before its ports are mapped, so every mapping the listing - * holds already has its app in the answer. + * One listing, of the mappings to this node's own address only - several + * nodes behind one router is an ordinary setup. A port mapUpnpPort has mapped + * since FluxOS started, or maps while this runs, is never removed. * * Everything - the listing and each removal - happens within `budgetMs`. The * listing has to finish early enough to leave time for removals, or nothing is @@ -391,24 +468,22 @@ function msUntilCleanupSlot(apiport, now = Date.now()) { * removal the router refuses with a UPnP error is stepped past; any other * failure - no answer in time, a connection refused or reset - ends the pass, * so as not to send more to a router that is not keeping up. - * @param {function(): Promise>} heldPorts every port something on this node holds + * @param {function(object[]): (object[]|Promise)} select the mappings to remove, from the listing * @param {number} budgetMs + * @param {string} what the mappings' name in the log * @returns {Promise} how many mappings were removed */ -async function removeStaleMappings(heldPorts, budgetMs) { +async function removeOwnMappings(select, budgetMs, what) { const deadline = monotonicMs() + budgetMs; const left = () => deadline - monotonicMs(); const listingMs = budgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, budgetMs / 2); const mappings = await client.getMappings({ local: true, signal: AbortSignal.timeout(listingMs) }); - const keep = new Set([...await heldPorts()].map(Number)); - const stale = mappings.filter((mapping) => Number.isInteger(mapping.public.port) - && !keep.has(mapping.public.port) - && isOwnedMappingDescription(mapping.description)); + const chosen = (await select(mappings)).filter((mapping) => Number.isInteger(mapping.public.port)); let removed = 0; // eslint-disable-next-line no-restricted-syntax - for (const [index, mapping] of stale.entries()) { + for (const [index, mapping] of chosen.entries()) { if (left() <= 0) { - log.warn(`UPnP - stale mapping cleanup out of time, ${stale.length - index} left for the next start`); + log.warn(`UPnP - ${what} cleanup out of time, ${chosen.length - index} left for the next start`); break; } const { host, port } = mapping.public; @@ -421,11 +496,11 @@ async function removeStaleMappings(heldPorts, budgetMs) { // eslint-disable-next-line no-await-in-loop await client.removeMapping({ public: { host, port }, protocol, signal: AbortSignal.timeout(left()) }); removed += 1; - log.info(`UPnP - stale mapping removed: ${protocol} ${port} (${mapping.description})`); + log.info(`UPnP - ${what} removed: ${protocol} ${port} (${mapping.description})`); } catch (error) { - log.warn(`UPnP - stale mapping ${protocol} ${port} (${mapping.description}) not removed: ${error.message}`); + log.warn(`UPnP - ${what} ${protocol} ${port} (${mapping.description}) not removed: ${error.message}`); if (!(error instanceof natUpnp.UpnpError)) { - log.warn(`UPnP - stale mapping cleanup stopped, ${stale.length - index - 1} left for the next start`); + log.warn(`UPnP - ${what} cleanup stopped, ${chosen.length - index - 1} left for the next start`); break; } } @@ -435,6 +510,45 @@ async function removeStaleMappings(heldPorts, budgetMs) { return removed; } +/** + * Removes the router's mappings to this node that this code made and nothing + * here holds any more: an app removed while the router was unreachable, a + * FluxOS that died between mapping and unmapping, a node reinstalled on the + * same address without the apps it ran, an api port that moved. They are made + * with an indefinite lease (or the router's longest, a week under IGDv2), so + * nothing else ever takes them, and home routers hold only so many. + * + * Only those with a description this code gives, on a port nothing here + * holds. What is held is asked for only after the router has been listed: an + * app's record is written before its ports are mapped, so every mapping the + * listing holds already has its app in the answer. + * @param {function(): Promise>} heldPorts every port something on this node holds + * @param {number} budgetMs + * @returns {Promise} how many mappings were removed + */ +async function removeStaleMappings(heldPorts, budgetMs) { + return removeOwnMappings(async (mappings) => { + const keep = new Set([...await heldPorts()].map(Number)); + return mappings.filter((mapping) => !keep.has(mapping.public.port) + && isOwnedMappingDescription(mapping.description)); + }, budgetMs, 'stale mapping'); +} + +/** + * Removes the router's test mappings to this node: a port test's, an app's + * prelaunch test's and verifyUPNPsupport's. A test removes its mapping before + * it ends, so while UPnP is being verified at the start of FluxOS every one the + * router holds was left by an earlier run. + * @returns {Promise} how many mappings were removed + */ +async function removeLeftoverTestMappings() { + return removeOwnMappings( + (mappings) => mappings.filter((mapping) => isTestMappingDescription(mapping.description)), + CLEANUP_SLOT_MS, + 'left-over test mapping', + ); +} + /** * To map a specified port and show a message if successfully mapped. Only accessible by admins and Flux team members. * @param {object} req Request. @@ -613,6 +727,7 @@ module.exports = { setupUPNP, mapUpnpPort, removeMapUpnpPort, + routerRefusals, corePorts, msUntilCleanupSlot, removeStaleMappings, diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 6014a1306e..8e4df80bb3 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -6,6 +6,14 @@ const { resetGlobalState } = require('./fixtures/globalState'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); +// what upnpService.mapUpnpPort answers for a port mapped, and for one the router refused +const MAPPED = { + mapped: true, code: null, reason: null, routerWide: false, +}; +const REFUSED = { + mapped: false, code: 728, reason: 'TCP 31000 not mapped: UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)', routerWide: true, +}; + // The full-install dockerService stub, shared by every proxyquire setup that // drives registerAppLocally. Pass overrides for the few tests that need a // specific return (e.g. a distinct getAppIdentifier or a shared pruneImages @@ -207,7 +215,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../utils/globalState': globalStateStub, '../../lib/log': logStub, @@ -851,7 +859,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../appRequirements/hwRequirements': hwRequirementsStub, '../appMessaging/messageVerifier': messageVerifierStub, @@ -1067,7 +1075,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../utils/globalState': globalStateStub, '../../lib/log': logStub, @@ -1273,7 +1281,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../utils/globalState': globalStateStub, '../../lib/log': logStub, @@ -1479,7 +1487,7 @@ describe('appInstaller tests', () => { }), }, '../pgpService': { decryptMessage: sinon.stub().resolves('user:token') }, - '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(true) }, + '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(MAPPED) }, '../utils/globalState': globalStateStub, '../utils/volumeService': { verifyAppVolumeMount: sinon.stub().resolves(), ensureMountPathsExist: sinon.stub().resolves() }, '../../lib/log': logStub, @@ -1565,7 +1573,7 @@ describe('appInstaller tests', () => { '../appManagement/appInspector': { startAppMonitoring: sinon.stub() }, '../utils/imageVerifier': { ImageVerifier: sinon.stub().returns({ addCredentials: sinon.stub(), verifyImage: sinon.stub().resolves(), throwIfError: sinon.stub(), supported: true, provider: 'docker.io' }) }, '../pgpService': { decryptMessage: sinon.stub().resolves('user:token') }, - '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(true) }, + '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(MAPPED) }, '../utils/enterpriseHelper': enterpriseHelperStub, '../utils/appSpecHelpers': appSpecHelpersStub, '../../lib/log': logStub, @@ -1610,7 +1618,7 @@ describe('appInstaller tests', () => { allowPort: sinon.stub().resolves({ status: true }), allowAppPorts: sinon.stub().resolves({ failed: [], locked: false }), }; - upnpStub = { isUPNP: sinon.stub().returns(true), mapUpnpPort: sinon.stub().resolves(true) }; + upnpStub = { isUPNP: sinon.stub().returns(true), mapUpnpPort: sinon.stub().resolves(MAPPED) }; logStub = { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }; appInstallerPorts = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { '../fluxNetworkHelper': firewallStub, @@ -1649,6 +1657,22 @@ describe('appInstaller tests', () => { expect(upnpStub.mapUpnpPort.calledWith(31000, 'Flux_App_myapp')).to.be.true; }); + it('fails the install with the router\'s reason when a port is refused', async () => { + upnpStub.mapUpnpPort.resolves(REFUSED); + + const error = await appInstallerPorts.setupApplicationPorts({ ports: [31000] }, 'myapp', false, null).catch((e) => e); + + expect(error.message).to.equal(`Error: Port 31000 FAILed to map. ${REFUSED.reason}`); + }); + + it('fails a v1 install with the router\'s reason when its port is refused', async () => { + upnpStub.mapUpnpPort.resolves(REFUSED); + + const error = await appInstallerPorts.setupApplicationPorts({ port: 31000 }, 'myapp', false, null).catch((e) => e); + + expect(error.message).to.equal(`Error: Port 31000 FAILed to map. ${REFUSED.reason}`); + }); + it('opens nothing for a test install', async () => { await appInstallerPorts.setupApplicationPorts({ ports: [31000] }, 'myapp', false, null, true); diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 255b111563..bae90ed14c 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai'); const sinon = require('sinon'); +const proxyquire = require('proxyquire'); // eslint-disable-next-line no-unused-vars const axios = require('axios'); const config = require('config'); @@ -12,6 +13,13 @@ const verificationHelper = require('../../ZelBack/src/services/verificationHelpe const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/daemonServiceMiscRpcs'); const upnpService = require('../../ZelBack/src/services/upnpService'); const networkStateService = require('../../ZelBack/src/services/networkStateService'); +const fluxHttpTestServer = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); + +const signer = { pubKey: 'pubkey', sign: () => 'signature' }; +// the check with a node that can sign its port test +const signingAvailabilityChecker = proxyquire('../../ZelBack/src/services/appMonitoring/availabilityChecker', { + '../utils/nodeSigner': { nodeSigner: async () => signer }, +}); describe('availabilityChecker tests', () => { let mockInstalledAppsFn; @@ -30,7 +38,7 @@ describe('availabilityChecker tests', () => { testingPort: null, nextTestingPort: null, originalPortFailed: null, - lastUPNPMapFailed: false, + routerRefusalsSeen: 0, }; mockPortsNotWorking = new Set(); mockFailedNodesCache = new Map(); @@ -320,29 +328,56 @@ describe('availabilityChecker tests', () => { sinon.assert.calledWith(delayStub, 15_000); }); - it('should handle UPNP mapping failures', async () => { - const apps = []; + describe('a node behind UPnP', () => { + const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; + let refusals; + let mapUpnpPortStub; + let removeMapUpnpPortStub; + let portTestStub; - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ - data: { synced: true }, + const refused = (code, routerWide) => ({ + mapped: false, code, reason: `TCP 1 not mapped: UPnP error ${code}`, routerWide, + }); + const MAPPED = { + mapped: true, code: null, reason: null, routerWide: false, + }; + // the router refuses `count` mappings of others - an install's, a restore's + const routerRefusesOthers = (count) => { refusals += count; }; + + beforeEach(() => { + refusals = 0; + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); + sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); + sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); + mockInstalledAppsFn.resolves({ status: 'success', data: [] }); + sinon.stub(upnpService, 'isUPNP').returns(true); + sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); + // The port under test is drawn at random, so this gate has to be closed + // too: left real, it refuses whenever the draw lands in upnpBannedPorts + // and the run returns before it ever reaches mapUpnpPort. + sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); + sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); + sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); + sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); + sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); + sinon.stub(upnpService, 'routerRefusals').callsFake(() => refusals); + mapUpnpPortStub = sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); + removeMapUpnpPortStub = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { + this.emit('listening'); + return this; + }); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'close').callsFake((callback) => callback && callback()); + portTestStub = sinon.stub(axios, 'post').resolves({ data: { status: 'success', data: { message: 'Ports reachable' } } }); }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: apps }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - // The port under test is drawn at random, so this gate has to be closed - // too: left real, it refuses whenever the draw lands in upnpBannedPorts - // and the run returns before it ever reaches mapUpnpPort. - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); // Failed - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); - await availabilityChecker.checkMyAppsAvailability( + // the test mapping refused as `result` gives it, one router-wide refusal counted when it is + const testMappingRefused = (result) => mapUpnpPortStub.callsFake(async () => { + if (result.routerWide) refusals += 1; + return result; + }); + + const pass = () => signingAvailabilityChecker.checkMyAppsAvailability( mockInstalledAppsFn, mockDosState, mockPortsNotWorking, @@ -350,118 +385,116 @@ describe('availabilityChecker tests', () => { isArcane, ); - expect(mockDosState.lastUPNPMapFailed).to.be.true; - }); + it('counts a test mapping the router refused for its own reason', async () => { + testMappingRefused(refused(728, true)); - it('opens the port under test to IPv4 only, and deletes only that rule', async () => { - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: [] }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - const allowTest = sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - const deleteTest = sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - const allowBoth = sinon.stub(fluxNetworkHelper, 'allowPort').resolves(); - const deleteBoth = sinon.stub(fluxNetworkHelper, 'deleteAllowPortRule').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); - - await availabilityChecker.checkMyAppsAvailability(mockInstalledAppsFn, mockDosState, mockPortsNotWorking, mockFailedNodesCache, isArcane); - - sinon.assert.calledOnceWithExactly(allowTest, mockDosState.testingPort); - sinon.assert.calledWith(deleteTest, mockDosState.testingPort); - sinon.assert.notCalled(allowBoth); - sinon.assert.notCalled(deleteBoth); - }); + await pass(); - it('should increase DOS state on repeated UPNP failures', async () => { - const apps = []; - mockDosState.lastUPNPMapFailed = true; // Already failed once + expect(mockDosState.portFailureScore).to.equal(4); + expect(mockDosState.dosStateValue, 'the node\'s DOS, which a count below the threshold leaves alone').to.equal(0); + sinon.assert.notCalled(portTestStub); + sinon.assert.calledWith(delayStub, 60_000); + }); - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ - data: { synced: true }, + it('does not count a test mapping refused for its port alone', async () => { + testMappingRefused(refused(718, false)); + + await pass(); + + expect(mockDosState.portFailureScore ?? 0).to.equal(0); }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: apps }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - // The port under test is drawn at random, so this gate has to be closed - // too: left real, it refuses whenever the draw lands in upnpBannedPorts - // and the run returns before it ever reaches mapUpnpPort. - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); - await availabilityChecker.checkMyAppsAvailability( - mockInstalledAppsFn, - mockDosState, - mockPortsNotWorking, - mockFailedNodesCache, - isArcane, - ); + it('removes what the router kept of a test mapping it refused', async () => { + testMappingRefused(refused(718, false)); - expect(mockDosState.portFailureScore).to.equal(4); - expect(mockDosState.dosStateValue, 'the node\'s DOS, which a count below the threshold leaves alone').to.equal(0); - }); + await pass(); - it('leaves another check\'s DOS message standing when its own UPnP failures reach the threshold', async () => { - mockDosState.lastUPNPMapFailed = true; - mockDosState.portFailureScore = 96; - mockDosState.dosStateValue = 12; - mockDosState.dosMessage = 'Flux is not available for outside communication'; + sinon.assert.calledOnceWithExactly(removeMapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App'); + }); - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: [] }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); + it('opens the port under test to IPv4 only, and deletes only that rule', async () => { + testMappingRefused(refused(728, true)); + const allowBoth = sinon.stub(fluxNetworkHelper, 'allowPort').resolves(); + const deleteBoth = sinon.stub(fluxNetworkHelper, 'deleteAllowPortRule').resolves(); - await availabilityChecker.checkMyAppsAvailability(mockInstalledAppsFn, mockDosState, mockPortsNotWorking, mockFailedNodesCache, isArcane); + await pass(); - expect(mockDosState.portFailureScore).to.equal(100); - expect(mockDosState.dosStateValue).to.equal(12); - expect(mockDosState.dosMessage).to.equal('Flux is not available for outside communication'); - }); + sinon.assert.calledOnceWithExactly(fluxNetworkHelper.allowTestPort, mockDosState.testingPort); + sinon.assert.calledWith(fluxNetworkHelper.deleteTestPortRule, mockDosState.testingPort); + sinon.assert.notCalled(allowBoth); + sinon.assert.notCalled(deleteBoth); + }); - it('holds the node out of service with its own message when its UPnP failures reach the threshold and no other stands', async () => { - mockDosState.lastUPNPMapFailed = true; - mockDosState.portFailureScore = 96; + it('counts once a pass however many mappings the router refused since the last', async () => { + routerRefusesOthers(7); - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: [] }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); + await pass(); + + expect(mockDosState.portFailureScore).to.equal(4); + expect(mockDosState.routerRefusalsSeen).to.equal(7); + }); - await availabilityChecker.checkMyAppsAvailability(mockInstalledAppsFn, mockDosState, mockPortsNotWorking, mockFailedNodesCache, isArcane); + it('counts nothing for refusals an earlier pass counted', async () => { + routerRefusesOthers(7); + mockDosState.routerRefusalsSeen = 7; - expect(mockDosState.dosMessage).to.match(/^Not possible to run applications on the node/); - expect(mockDosState.dosStateValue).to.equal(100); + await pass(); + + expect(mockDosState.portFailureScore ?? 0).to.equal(0); + }); + + it('keeps its router DOS through a pass whose test worked while the router refused others', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + routerRefusesOthers(1); + + await pass(); + + sinon.assert.calledOnce(portTestStub); + expect(mockDosState.dosMessage).to.equal(UPNP_DOS); + expect(mockDosState.dosStateValue).to.equal(100); + expect(mockDosState.portFailureScore).to.equal(100); + }); + + it('ends its router DOS on a pass whose test worked and the router refused nothing', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + routerRefusesOthers(3); + mockDosState.routerRefusalsSeen = 3; + + await pass(); + + sinon.assert.calledOnce(portTestStub); + expect(mockDosState.dosMessage).to.equal(null); + expect(mockDosState.dosStateValue).to.equal(0); + expect(mockDosState.portFailureScore).to.equal(0); + }); + + it('leaves another check\'s DOS message standing when its own UPnP failures reach the threshold', async () => { + mockDosState.portFailureScore = 96; + mockDosState.dosStateValue = 12; + mockDosState.dosMessage = 'Flux is not available for outside communication'; + testMappingRefused(refused(728, true)); + + await pass(); + + expect(mockDosState.portFailureScore).to.equal(100); + expect(mockDosState.dosStateValue).to.equal(12); + expect(mockDosState.dosMessage).to.equal('Flux is not available for outside communication'); + }); + + it('holds the node out of service with its own message when its UPnP failures reach the threshold and no other stands', async () => { + mockDosState.portFailureScore = 96; + testMappingRefused(refused(728, true)); + + await pass(); + + expect(mockDosState.dosMessage).to.equal(UPNP_DOS); + expect(mockDosState.dosStateValue).to.equal(100); + sinon.assert.calledWith(delayStub, 300_000); + }); }); it('should handle errors gracefully and retry', async () => { diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 3ef3f02239..ee85aa3c6e 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -80,7 +80,9 @@ describe('fluxNetworkHelper tests', () => { beforeEach(() => { sinon.stub(upnpService, 'isUPNP').returns(false); sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); - sinon.stub(upnpService, 'mapUpnpPort').resolves(true); + sinon.stub(upnpService, 'mapUpnpPort').resolves({ + mapped: true, code: null, reason: null, routerWide: false, + }); // The installed copy of the ufw helper is the one to run (utils/ufwHelper has its own tests). sinon.stub(ufwHelper, 'path').resolves(ufwHelper.UFW_HELPER); }); diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index 08078917e1..a976c56722 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -17,6 +17,14 @@ const appQueryService = require('../../ZelBack/src/services/appQuery/appQuerySer const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); const globalState = require('../../ZelBack/src/services/utils/globalState'); +// what upnpService.mapUpnpPort answers for a port mapped, and for one the router refused +const MAPPED = { + mapped: true, code: null, reason: null, routerWide: false, +}; +const REFUSED = { + mapped: false, code: 728, reason: 'TCP 31000 not mapped: UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)', routerWide: true, +}; + describe('portManager tests', () => { before(requireMongo); @@ -606,7 +614,7 @@ describe('portManager tests', () => { sinon.stub(upnpService, 'isUPNP').returns(false); sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(false); sinon.stub(fluxNetworkHelper, 'allowPort').resolves(true); - sinon.stub(upnpService, 'mapUpnpPort').resolves(true); + sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); sinon.stub(serviceHelper, 'delay').resolves(); sinon.stub(appUninstaller, 'removeAppLocally').resolves(); portManager.upnpMapFailures.clear(); @@ -643,7 +651,7 @@ describe('portManager tests', () => { // removeAppLocally(force, sendMessage) - a transient router blip nuked // a running app and broadcast its removal to the network upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); @@ -653,8 +661,8 @@ describe('portManager tests', () => { it('should retry a failed port within the cycle and record no failure on recovery', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.onFirstCall().resolves(false); - upnpService.mapUpnpPort.resolves(true); + upnpService.mapUpnpPort.onFirstCall().resolves(REFUSED); + upnpService.mapUpnpPort.resolves(MAPPED); await portManager.restoreAppsPortsSupport(); @@ -664,7 +672,7 @@ describe('portManager tests', () => { it('should not remove before the sustained window even after enough failing cycles', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); await portManager.restoreAppsPortsSupport(); @@ -677,7 +685,7 @@ describe('portManager tests', () => { it('should remove and broadcast only after sustained failure (cycles AND window)', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); const nowMonotonicMs = Number(process.hrtime.bigint() / 1000000n); portManager.upnpMapFailures.set('App1', { cycles: 2, @@ -696,7 +704,7 @@ describe('portManager tests', () => { { name: 'App2', version: 3, ports: [30002] }, ]); upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); @@ -710,11 +718,11 @@ describe('portManager tests', () => { it('should clear the failure tracker once mapping succeeds again', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); expect(portManager.upnpMapFailures.get('App1').cycles).to.equal(1); - upnpService.mapUpnpPort.resolves(true); + upnpService.mapUpnpPort.resolves(MAPPED); await portManager.restoreAppsPortsSupport(); expect(portManager.upnpMapFailures.has('App1')).to.be.false; @@ -1358,4 +1366,16 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(result.reason).to.equal('noOtherObserver'); expect(result.peers).to.have.length(1); }); + + it('asks no peer, and removes what the router kept, when the router refuses the port under test', async () => { + upnpService.isUPNP.returns(true); + sinon.stub(upnpService, 'mapUpnpPort').resolves(REFUSED); + const removeMapping = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); + + const result = await withPeers([UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE]); + + expect(result).to.deep.equal({ ok: false, reason: 'error' }); + sinon.assert.notCalled(axios.post); + sinon.assert.calledWith(removeMapping, port, `Flux_Prelaunch_App_${port}`); + }); }); diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index a65332f130..3e6b806815 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -223,7 +223,7 @@ describe('upnpService tests', () => { it('should log a proper error if createMapping throws', async () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'createMapping').throws(); + sinon.stub(natUpnp.Client.prototype, 'createMapping').rejects(new Error('connect ECONNREFUSED')); sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); @@ -235,8 +235,78 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping'); + sinon.assert.calledOnceWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping: TCP 16130 not mapped: connect ECONNREFUSED'); + }); + + describe('when the router\'s table is full', () => { + let createMappingStub; + let getMappingsStub; + let removeMappingStub; + + const listed = (port, description) => ({ + ...testMapping, public: { host: '', port }, private: { host: '192.168.1.10', port }, description, + }); + const tableFull = () => new natUpnp.UpnpError(728, 'NoPortMapsAvailable', 'AddPortMapping'); + + beforeEach(() => { + upnpService.mappedHere.clear(); + sinon.stub(serviceHelper, 'delay').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves('1.2.3.4'); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping'); + createMappingStub.onFirstCall().rejects(tableFull()); + createMappingStub.resolves(); + // the lookup after the refusal finds no entry; the one after the mapping finds it + sinon.stub(natUpnp.Client.prototype, 'getMapping').onFirstCall().resolves(null).resolves(testMapping); + getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([ + listed(31000, 'Flux_Test_App'), + listed(31001, 'Flux_Prelaunch_App_31001'), + listed(31002, 'Flux_App_web'), + listed(31003, 'Flux_Backend_API'), + // this check's own, held while it runs + listed(16130, 'Flux_UPNP_Mapping_Test'), + ]); + removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + }); + + it('should remove the test mappings earlier runs left, and nothing else, then map once more', async () => { + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(true); + sinon.assert.calledOnceWithExactly(getMappingsStub, { local: true, signal: sinon.match.instanceOf(AbortSignal) }); + const removedPorts = removeMappingStub.getCalls().map((call) => call.args[0].public.port ?? call.args[0].public); + expect(removedPorts).to.deep.equal([31000, 31001, 16130]); + sinon.assert.calledTwice(createMappingStub); + }); + + it('should fail without mapping again when there is no test mapping to remove', async () => { + getMappingsStub.resolves([listed(31002, 'Flux_App_web')]); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.calledOnce(createMappingStub); + sinon.assert.calledWithMatch(logSpy, /^VerifyUPNPsupport - Failed Create Mapping: TCP 16130 not mapped: UPnP error 728/); + }); + + it('should fail without mapping again when the router cannot be listed', async () => { + getMappingsStub.rejects(new Error('listing timed out')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.calledOnce(createMappingStub); + }); + + it('should list nothing when the router refuses for any other reason', async () => { + createMappingStub.onFirstCall().rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.notCalled(getMappingsStub); + sinon.assert.calledOnce(createMappingStub); + }); }); it('should look up the test mapping it made, by its port, and nothing else', async () => { @@ -458,16 +528,16 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); sinon.assert.callCount(createMappingSpy, 4); sinon.assert.calledWithExactly(createMappingSpy, { - public: 123, private: 123, ttl: 0, description: 'Flux_Backend_API', + public: 123, private: 123, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 124, private: 124, ttl: 0, description: 'Flux_Backend_API_SSL', + public: 124, private: 124, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API_SSL', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 122, private: 122, ttl: 0, description: 'Flux_Home_UI', + public: 122, private: 122, ttl: 0, protocol: 'TCP', description: 'Flux_Home_UI', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 125, private: 125, ttl: 0, description: 'Flux_Syncthing', + public: 125, private: 125, ttl: 0, protocol: 'TCP', description: 'Flux_Syncthing', }); }); @@ -485,19 +555,41 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); sinon.assert.callCount(createMappingSpy, 4); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16127, private: 16127, ttl: 0, description: 'Flux_Backend_API', + public: 16127, private: 16127, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16128, private: 16128, ttl: 0, description: 'Flux_Backend_API_SSL', + public: 16128, private: 16128, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API_SSL', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16126, private: 16126, ttl: 0, description: 'Flux_Home_UI', + public: 16126, private: 16126, ttl: 0, protocol: 'TCP', description: 'Flux_Home_UI', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16129, private: 16129, ttl: 0, description: 'Flux_Syncthing', + public: 16129, private: 16129, ttl: 0, protocol: 'TCP', description: 'Flux_Syncthing', }); }); + it('should take a core port the router refuses but already maps to this node as mapped', async () => { + createMappingSpy.withArgs(sinon.match({ public: 124 })).rejects(new natUpnp.UpnpError(728, 'NoPortMapsAvailable', 'AddPortMapping')); + getMappingStub.withArgs({ public: 124, protocol: 'TCP' }).resolves(fluxadmMapping({ + public: { host: '', port: 124 }, private: { host: '192.168.1.10', port: 124 }, description: 'Flux_Backend_API_SSL', enabled: true, + })); + + const result = await runSetup(123); + + expect(result).to.equal(true); + sinon.assert.notCalled(logSpy); + }); + + it('should fail on a core port the router refuses, naming it and the router\'s reason', async () => { + createMappingSpy.withArgs(sinon.match({ public: 124 })).rejects(new natUpnp.UpnpError(728, 'NoPortMapsAvailable', 'AddPortMapping')); + + const result = await runSetup(123); + + expect(result).to.equal(false); + sinon.assert.calledOnceWithExactly(logSpy, 'setupUPNP - TCP 124 not mapped: UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)'); + sinon.assert.callCount(createMappingSpy, 2); + }); + it('should return error if client response throws', async () => { createMappingSpy.throws(); @@ -515,103 +607,257 @@ describe('upnpService tests', () => { describe('mapUpnpPort tests', () => { let logSpy; - let createMappingSpy; + let createMappingStub; + let getMappingStub; + + const refusal = (code, description) => new natUpnp.UpnpError(code, description, 'AddPortMapping'); + // the router's entry for port 123, to this node unless overridden + const entry = (overrides = {}) => ({ + public: { host: '', port: 123 }, + private: { host: '192.168.1.10', port: 123 }, + protocol: 'udp', + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + ...overrides, + }); beforeEach(() => { + upnpService.mappedHere.clear(); logSpy = sinon.spy(log, 'error'); - createMappingSpy = sinon.stub(natUpnp.Client.prototype, 'createMapping'); + sinon.stub(serviceHelper, 'delay').resolves(); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); }); afterEach(() => { sinon.restore(); }); - it('should return true if all client responses are valid', async () => { - createMappingSpy.returns(true); + it('should map TCP and then UDP, port for port, with no lease', async () => { + const result = await upnpService.mapUpnpPort(123, 'some description'); - const clock = sinon.useFakeTimers(); + expect(result).to.deep.equal({ + mapped: true, code: null, reason: null, routerWide: false, + }); + sinon.assert.notCalled(logSpy); + sinon.assert.calledTwice(createMappingStub); + sinon.assert.calledWithExactly(createMappingStub.firstCall, { + public: 123, private: 123, ttl: 0, protocol: 'TCP', description: 'some description', + }); + sinon.assert.calledWithExactly(createMappingStub.secondCall, { + public: 123, private: 123, ttl: 0, protocol: 'UDP', description: 'some description', + }); + }); - const promise = upnpService.mapUpnpPort(123, 'some description'); + it('should not ask for UDP when TCP is refused', async () => { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); - await clock.tickAsync(1_000); - const result = await promise; + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); - expect(result).to.equal(true); - sinon.assert.notCalled(logSpy); - sinon.assert.calledTwice(createMappingSpy); - sinon.assert.calledWithExactly(createMappingSpy, { - public: 123, - private: 123, - ttl: 0, - protocol: 'TCP', - description: 'some description', + expect(result.mapped).to.equal(false); + sinon.assert.calledOnce(createMappingStub); + }); + + it('should give a port another machine holds as the port\'s refusal, not the router\'s', async () => { + createMappingStub.onSecondCall().rejects(refusal(718, 'ConflictInMappingEntry')); + getMappingStub.resolves(entry({ local: false, private: { host: '192.168.1.11', port: 123 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.deep.equal({ + mapped: false, + code: 718, + reason: 'UDP 123 not mapped: UPnP error 718: ConflictInMappingEntry (action: AddPortMapping)', + routerWide: false, }); - sinon.assert.calledWithExactly(createMappingSpy, { - public: 123, - private: 123, - ttl: 0, - protocol: 'UDP', - description: 'some description', + expect(upnpService.routerRefusals()).to.equal(before); + sinon.assert.calledOnceWithExactly(logSpy, `UPnP - ${result.reason}`); + }); + + it('should count a full table as the router\'s refusal', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 728, routerWide: true }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + }); + + it('should count a router that does not answer as the router\'s refusal, and not ask it again', async () => { + createMappingStub.rejects(new Error('timeout of 10000ms exceeded')); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.deep.equal({ + mapped: false, code: null, reason: 'TCP 123 not mapped: timeout of 10000ms exceeded', routerWide: true, }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + sinon.assert.notCalled(getMappingStub); }); - it('should return error if client response throws', async () => { - createMappingSpy.throws(); + it('should count no other refusal as the router\'s', async () => { + createMappingStub.rejects(refusal(501, 'ActionFailed')); + const before = upnpService.routerRefusals(); - const clock = sinon.useFakeTimers(); + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); - const promise = upnpService.mapUpnpPort(123); + expect(result).to.include({ mapped: false, code: 501, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); - await clock.tickAsync(1_000); - const result = await promise; + it('should take a refusal as mapped when the router already maps the port to this node', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.callsFake(async ({ protocol }) => entry({ protocol: protocol.toLowerCase() })); + const before = upnpService.routerRefusals(); - expect(result).to.equal(false); - sinon.assert.calledOnce(logSpy); + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result.mapped).to.equal(true); + expect(upnpService.routerRefusals()).to.equal(before); + sinon.assert.calledWithExactly(getMappingStub.firstCall, { public: 123, protocol: 'TCP' }); + sinon.assert.calledWithExactly(getMappingStub.secondCall, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(logSpy); + }); + + [ + ['to another port of this node', { private: { host: '192.168.1.10', port: 124 } }], + ['disabled', { enabled: false }], + ['where this node cannot tell whose it is', { local: null }], + ].forEach(([what, overrides]) => { + it(`should keep the refusal when the router's entry is ${what}`, async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.resolves(entry(overrides)); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 728 }); + }); + }); + + it('should keep the refusal when the router cannot be asked for its entry', async () => { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); + getMappingStub.rejects(new Error('socket hang up')); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 718 }); + }); + + it('should ask the router nothing for what is not a port, and count nothing', async () => { + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(Number.NaN, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: null, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + sinon.assert.notCalled(createMappingStub); + }); + + it('should no longer hold a port it failed to map', async () => { + createMappingStub.onSecondCall().rejects(refusal(718, 'ConflictInMappingEntry')); + + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(upnpService.mappedHere.has(123)).to.equal(false); + }); + + it('should hold a port it mapped', async () => { + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(upnpService.mappedHere.has(123)).to.equal(true); }); }); describe('removeMapUpnpPort tests', () => { let logSpy; - let removeMappingSpy; + let getMappingStub; + let removeMappingStub; + + const entry = (protocol, overrides = {}) => ({ + public: { host: '', port: 123 }, + private: { host: '192.168.1.10', port: 123 }, + protocol, + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + ...overrides, + }); + const removedProtocols = () => removeMappingStub.getCalls().map((call) => call.args[0].protocol); beforeEach(() => { logSpy = sinon.spy(log, 'error'); - removeMappingSpy = sinon.stub(natUpnp.Client.prototype, 'removeMapping'); + sinon.stub(serviceHelper, 'delay').resolves(); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping') + .callsFake(async ({ protocol }) => entry(protocol.toLowerCase())); + removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); afterEach(() => { sinon.restore(); }); - it('should return true if all client responses are valid', async () => { - removeMappingSpy.returns(true); + it('should remove both protocols of a port the router maps to this node', async () => { + const result = await upnpService.removeMapUpnpPort(123); - const clock = sinon.useFakeTimers(); + expect(result).to.equal(true); + sinon.assert.notCalled(logSpy); + sinon.assert.calledWithExactly(getMappingStub, { public: 123, protocol: 'TCP' }); + sinon.assert.calledWithExactly(getMappingStub, { public: 123, protocol: 'UDP' }); + sinon.assert.calledWithExactly(removeMappingStub.firstCall, { public: 123, protocol: 'TCP' }); + sinon.assert.calledWithExactly(removeMappingStub.secondCall, { public: 123, protocol: 'UDP' }); + }); - const promise = upnpService.removeMapUpnpPort(123, 'some description'); + it('should leave a protocol the router maps to another machine', async () => { + getMappingStub.withArgs({ public: 123, protocol: 'UDP' }) + .resolves(entry('udp', { local: false, private: { host: '192.168.1.11', port: 123 } })); - await clock.tickAsync(1_000); - const result = await promise; + const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(true); - sinon.assert.notCalled(logSpy); - sinon.assert.calledTwice(removeMappingSpy); - sinon.assert.calledWithExactly(removeMappingSpy, { public: 123, protocol: 'TCP' }); - sinon.assert.calledWithExactly(removeMappingSpy, { public: 123, protocol: 'UDP' }); + expect(removedProtocols()).to.deep.equal(['TCP']); }); - it('should return error if client response throws', async () => { - removeMappingSpy.throws(); + it('should remove nothing of a protocol the router does not hold', async () => { + getMappingStub.withArgs({ public: 123, protocol: 'TCP' }).resolves(null); - const clock = sinon.useFakeTimers(); + const result = await upnpService.removeMapUpnpPort(123); - const promise = upnpService.removeMapUpnpPort(123); + expect(result).to.equal(true); + expect(removedProtocols()).to.deep.equal(['UDP']); + }); - await clock.tickAsync(1_000); - const result = await promise; + it('should remove a protocol whose entry cannot be looked up', async () => { + getMappingStub.rejects(new Error('socket hang up')); + + const result = await upnpService.removeMapUpnpPort(123); + + expect(result).to.equal(true); + expect(removedProtocols()).to.deep.equal(['TCP', 'UDP']); + }); + + it('should remove UDP when TCP is refused, and report the port not removed', async () => { + removeMappingStub.withArgs({ public: 123, protocol: 'TCP' }).rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping')); + + const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(false); - sinon.assert.calledOnce(logSpy); + expect(removedProtocols()).to.deep.equal(['TCP', 'UDP']); + sinon.assert.calledOnceWithExactly(logSpy, 'UPnP - TCP 123 not unmapped: UPnP error 501: ActionFailed (action: DeletePortMapping)'); + }); + + it('should take a removal the router has no entry for as removed', async () => { + removeMappingStub.rejects(new natUpnp.UpnpError(714, 'NoSuchEntryInArray', 'DeletePortMapping')); + + const result = await upnpService.removeMapUpnpPort(123); + + expect(result).to.equal(true); + sinon.assert.notCalled(logSpy); }); }); @@ -833,6 +1079,7 @@ describe('upnpService tests', () => { it('should remove a port mapUpnpPort mapped once removeMapUpnpPort has given it up, even if the router refused', async () => { sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(mapping(31000, 'Flux_App_gone')); await upnpService.mapUpnpPort(31000, 'Flux_App_gone'); removeMappingStub.onFirstCall().rejects(refused()); await upnpService.removeMapUpnpPort(31000); From 06c3502d4caf2760075105c730ffeb229258ee39 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 17:35:43 +0100 Subject: [PATCH 007/369] test(harness): 3001, a node behind a full router table verifies UPnP The stub gateway refuses what a router's table does: a port another machine holds (718), and a new entry once it holds as many as its capacity (728). A suite gives a gateway its capacity and the mappings it holds already. 3001 puts a node on a custom api port behind a table full of its own left-over test mappings and one mapping of another machine. The node removes the left-overs, verifies, and maps its four ports in their room; the other machine's mapping stays as it was. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 24 +++++- .../runner/framework/external-http-control.js | 11 ++- ...ehind-a-full-router-table-verifies-upnp.js | 83 +++++++++++++++++++ 3 files changed, 111 insertions(+), 7 deletions(-) create mode 100644 test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 3c0b94abbf..0a76ea641e 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -392,7 +392,8 @@ function imageSyncthingVersion() { const state = { // Nodes behind a UPnP gateway, by fleet address: the external address it reports (the - // node's own when null), its port mappings, and every action the node sent it. + // node's own when null), how many mappings its table holds (no limit when null), its port + // mappings, and every action the node sent it. upnpGateways: {}, // Bytes served INSTEAD of the signed artifact, under the signed artifact's own name. A // mirror or CDN answering correctly-named requests with the wrong content, which is the @@ -716,7 +717,9 @@ app.get('/repos/:owner/:repo', (req, res) => { // // A node behind a gateway is served one: a WANIPConnection service whose control URL answers // the actions upnpService sends - the external address, and adding, looking up, listing and -// deleting port mappings - from that node's own mapping table. +// deleting port mappings - from that node's own mapping table. Its table refuses what a +// router's does: a port another machine holds (718), and a new entry once it holds as many as +// its capacity (728). const UPNP_SERVICE = 'urn:schemas-upnp-org:service:WANIPConnection:1'; function upnpGatewayOf(req) { @@ -771,6 +774,15 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { soapReply(res, action, { NewExternalIPAddress: gateway.externalIp ?? clientAddress(req) }); return; case 'AddPortMapping': { + const held = gateway.mappings.find(sameEntry); + if (held && held.internalClient !== arg('NewInternalClient')) { + soapFault(res, 718, 'ConflictInMappingEntry'); + return; + } + if (!held && gateway.capacity !== null && gateway.mappings.length >= gateway.capacity) { + soapFault(res, 728, 'NoPortMapsAvailable'); + return; + } gateway.mappings = gateway.mappings.filter((m) => !sameEntry(m)); gateway.mappings.push({ externalPort: Number(arg('NewExternalPort')), @@ -999,12 +1011,16 @@ control.post('/storage-requests/reset', (req, res) => { // Put a node behind a UPnP gateway, with no mappings yet. control.post('/upnp-gateway', (req, res) => { - const { ip, externalIp = null } = req.body || {}; + const { + ip, externalIp = null, capacity = null, mappings = [], + } = req.body || {}; if (!ip) { res.status(400).json({ error: 'ip is required' }); return; } - state.upnpGateways[ip] = { externalIp, mappings: [], requests: [] }; + state.upnpGateways[ip] = { + externalIp, capacity, mappings, requests: [], + }; res.json({ ok: true }); }); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index aaa7c23749..c172b6601e 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -191,10 +191,15 @@ export async function resetStorageRequests() { * Put a node behind a UPnP gateway the stub serves it. Every other node is served * a gateway with no WAN service, and runs without UPnP. * @param {string} ip - the node's fleet address, as subnet-config names it - * @param {{externalIp?: string}} [opts] - the address the gateway reports; the node's own when absent + * @param {object} [opts] + * @param {string} [opts.externalIp] - the address the gateway reports; the node's own when absent + * @param {number} [opts.capacity] - how many mappings its table holds; no limit when absent + * @param {object[]} [opts.mappings] - what its table holds already, as upnpGatewayState lists it */ -export async function enableUpnpGateway(ip, { externalIp = null } = {}) { - const res = await post('/upnp-gateway', { ip, externalIp }); +export async function enableUpnpGateway(ip, { externalIp = null, capacity = null, mappings = [] } = {}) { + const res = await post('/upnp-gateway', { + ip, externalIp, capacity, mappings, + }); if (!res.ok) throw new Error(`enableUpnpGateway: ${JSON.stringify(res)}`); } diff --git a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js new file mode 100644 index 0000000000..3e32671b64 --- /dev/null +++ b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js @@ -0,0 +1,83 @@ +// weight: light +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const subnet = getSubnetConfig(); + +// A node on any api port but 16127 runs only if it verifies UPnP, and it verifies by mapping +// a test port. Its router's table is full: the test mappings earlier runs of FluxOS left on +// it, and one mapping of another machine on the network. Until UPnP is verified the stale +// mapping cleanup never runs, so the verification itself makes room: it removes this node's +// left-over test mappings, maps its test port, and the node maps its own ports in the room +// that leaves. The other machine's mapping is not this node's to remove. + +const API_PORT = 16137; +// The node maps no other address: upnp.nodeIp is empty in the harness, so its mappings +// point at the address the client is given in its place. +const NODE_CLIENT = '127.0.0.1'; +const OTHER_MACHINE = subnet.nodeIp(9); + +const mapping = (externalPort, description, internalClient = NODE_CLIENT) => ({ + externalPort, protocol: 'TCP', internalPort: externalPort, internalClient, description, leaseDuration: 0, +}); +const LEFT_OVER = [ + mapping(31000, 'Flux_Test_App'), + mapping(31001, 'Flux_Prelaunch_App_31001'), + mapping(31002, 'Flux_Test_App'), + mapping(31003, 'Flux_UPNP_Mapping_Test'), +]; +const OTHERS = mapping(32000, 'Flux_App_sibling', OTHER_MACHINE); +// Full as it starts, and room for the node's four ports once the left-overs are gone. +const CAPACITY = LEFT_OVER.length + 1; +if (CAPACITY - 1 < 4) throw new Error('3001: the table must have room for the four core ports once cleared'); + +describe('a node behind a full router table verifies UPnP', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + await enableUpnpGateway(nodeIp, { capacity: CAPACITY, mappings: [...LEFT_OVER, OTHERS] }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('removes its left-over test mappings and maps its own ports in their room', async function () { + this.timeout(180000); + const corePorts = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2]; + let table = []; + await waitFor(async () => { + table = (await upnpGatewayState(nodeIp)).mappings; + return corePorts.every((port) => table.some((m) => m.externalPort === port)); + }, { timeout: 150000, interval: 2000, label: 'the node\'s own ports mapped' }).catch(() => {}); + + expect(table.map((m) => m.externalPort).sort(), 'the router\'s table').to.deep.equal([...corePorts, OTHERS.externalPort].sort()); + LEFT_OVER.forEach(({ externalPort, description }) => { + expect(env.nodeHasLog(0, `UPnP - left-over test mapping removed: TCP ${externalPort} (${description})`), `${externalPort} removed`).to.equal(true); + }); + }); + + it('leaves the other machine\'s mapping as it was', async function () { + const { mappings, requests } = await upnpGatewayState(nodeIp); + + expect(requests.some((r) => r.action === 'GetGenericPortMappingEntry'), 'canary: the node listed the table').to.equal(true); + expect(mappings.find((m) => m.externalPort === OTHERS.externalPort)).to.deep.equal(OTHERS); + }); +}); From dc70b24ae8548ec3f56c5ab2e43d0b31be4bfb10 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 17:39:49 +0100 Subject: [PATCH 008/369] test(harness): 3001 reads what the node removed from the router's table, not its log A FluxOS restarted in the harness writes no more to the log the suite reads. Co-Authored-By: Claude Opus 5.5 --- .../3001-a-node-behind-a-full-router-table-verifies-upnp.js | 3 --- 1 file changed, 3 deletions(-) diff --git a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js index 3e32671b64..ceeda39457 100644 --- a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js +++ b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js @@ -69,9 +69,6 @@ describe('a node behind a full router table verifies UPnP', function () { }, { timeout: 150000, interval: 2000, label: 'the node\'s own ports mapped' }).catch(() => {}); expect(table.map((m) => m.externalPort).sort(), 'the router\'s table').to.deep.equal([...corePorts, OTHERS.externalPort].sort()); - LEFT_OVER.forEach(({ externalPort, description }) => { - expect(env.nodeHasLog(0, `UPnP - left-over test mapping removed: TCP ${externalPort} (${description})`), `${externalPort} removed`).to.equal(true); - }); }); it('leaves the other machine\'s mapping as it was', async function () { From dba8604546cc5906be5b36ce2188a710f26614bb Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 17:57:05 +0100 Subject: [PATCH 009/369] test(harness): 2226 moves the node only once every node holds its app at the old port A move carries only the rows standing at the address it leaves, and a peer refuses one from an address where it holds none. The node broadcast its app 187 ms before the restart killed it, so no peer stored the rows and every peer refused the move. Co-Authored-By: Claude Opus 5.5 --- .../2226-a-node-whose-api-port-changes-keeps-its-apps.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js b/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js index 5d84defbce..6b60712e2a 100644 --- a/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js +++ b/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js @@ -98,6 +98,13 @@ describe('a node whose api port changes keeps its apps', function () { idx = await seedAndInstall(env, app); nodeIp = subnet.nodeIp(idx + 1); await waitFor(isUp, { timeout: 120000, interval: 3000, label: 'the app running before the port moves' }); + // A move carries only the rows standing at the address it leaves: a peer that does + // not yet hold the app there refuses it. + const oldSocket = `${nodeIp}:${OLD_PORT}`; + const holdsAtOldPort = async (i) => (await dbClient(i + 1).getAppLocations(appName)).some((row) => row.ip === oldSocket); + await waitFor(async () => (await Promise.all([0, 1, 2].map(holdsAtOldPort))).every(Boolean), { + timeout: 120000, interval: 3000, label: 'every node holds the app at the old port', + }); }); after(async function () { From 8aadc53adc87635371b1d50a44ea9ee8299e4b99 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 19:15:25 +0100 Subject: [PATCH 010/369] fix(upnp): every refusal that is not the port's own counts as the router's Routers do not agree on how they refuse a mapping when their table is full: some answer 728 NoPortMapsAvailable, others a generic 501 Action Failed, which some also give for every refresh of an entry they hold. A full table cannot be told by its code, so only a refusal that names the port or the request as the problem is the port's - 402, 716, 718, 724, 725, 726, 727, 729 - and every other refusal counts in routerRefusals(). verifyUPNPsupport removes the left-over test mappings on any refusal of the router's own that the router gave, not on 728 alone. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 29 +++++++++----- tests/unit/upnpService.test.js | 60 +++++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 16 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 0bed2b8adc..aaa3e053b1 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -52,10 +52,19 @@ const CLEANUP_REMOVAL_RESERVE_MS = 15 * 1000; const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); -// The router's answers this code acts on: a full table, and no such entry. -const NO_PORT_MAPS_AVAILABLE = 728; const NO_SUCH_ENTRY = 714; +// The refusals of a mapping that name the port or the request as the problem: +// invalid arguments (402), a wildcard or a pair of ports the router will not +// take (716, 724, 726, 727), a lease it will not grant (725), a port another +// machine or another mechanism holds (718, 729). Every other refusal is the +// router's own. Routers do not agree on how they refuse a mapping when their +// table is full: some answer 728 NoPortMapsAvailable, as the standard has it, +// others a generic 501 Action Failed, which some also give for every refresh +// of an entry they hold. A full table cannot be told by its code, so a refusal +// that is not the port's is taken as the router's. +const PORT_REFUSALS = new Set([402, 716, 718, 724, 725, 726, 727, 729]); + // What a test leaves on the router while it runs and removes before it ends. const TEST_MAPPING_DESCRIPTIONS = new Set(['Flux_UPNP_Mapping_Test', 'Flux_Test_App']); const TEST_MAPPING_PREFIX = 'Flux_Prelaunch_App_'; @@ -68,9 +77,8 @@ let routerRefusalCount = 0; /** * How many mappings the router has refused since FluxOS started for a reason - * that is the router's own and not the port's: a full table, or no answer. A - * port another machine holds (718) and every other refusal are the port's. - * Only ever goes up. + * that is the router's own and not the port's: no answer, or any refusal but + * those PORT_REFUSALS names. Only ever goes up. * @returns {number} */ function routerRefusals() { @@ -126,7 +134,8 @@ async function mapProtocol(port, protocol, description) { } } const code = answered ? error.code : null; - const routerWide = !answered || code === NO_PORT_MAPS_AVAILABLE; + // no answer, with no code, is the router's too + const routerWide = !PORT_REFUSALS.has(code); if (routerWide) routerRefusalCount += 1; return { mapped: false, code, reason: `${protocol} ${port} not mapped: ${error.message}`, routerWide, @@ -214,9 +223,11 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { // router IP exits on that. A test that fails leaves it held. mappedHere.add(+apiport + 3); let testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test'); - // a full table: until UPnP is verified the stale mapping cleanup never runs, - // so the test mappings earlier runs left are removed here to make room - if (testMapping.code === NO_PORT_MAPS_AVAILABLE) { + // refused for a reason of the router's own, as a full table is: until UPnP is + // verified the stale mapping cleanup never runs, so the test mappings earlier + // runs left are removed here to make room. A router that did not answer is + // not listed. + if (testMapping.routerWide && testMapping.code !== null) { let removed = 0; try { removed = await removeLeftoverTestMappings(); diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 3e6b806815..f64c571da4 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -298,7 +298,27 @@ describe('upnpService tests', () => { sinon.assert.calledOnce(createMappingStub); }); - it('should list nothing when the router refuses for any other reason', async () => { + it('should remove the left-overs, and map once more, when the router refuses with a generic 501 Action Failed', async () => { + createMappingStub.onFirstCall().rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'AddPortMapping')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(true); + sinon.assert.calledOnce(getMappingsStub); + sinon.assert.calledTwice(createMappingStub); + }); + + it('should list nothing when the router does not answer', async () => { + createMappingStub.onFirstCall().rejects(new Error('timeout of 10000ms exceeded')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.notCalled(getMappingsStub); + sinon.assert.calledOnce(createMappingStub); + }); + + it('should list nothing when the router refuses for the port\'s own reason', async () => { createMappingStub.onFirstCall().rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); const result = await upnpService.verifyUPNPsupport(); @@ -700,14 +720,40 @@ describe('upnpService tests', () => { sinon.assert.notCalled(getMappingStub); }); - it('should count no other refusal as the router\'s', async () => { - createMappingStub.rejects(refusal(501, 'ActionFailed')); - const before = upnpService.routerRefusals(); + [ + [501, 'ActionFailed'], + [606, 'ActionNotAuthorized'], + ].forEach(([code, description]) => { + it(`should count a ${code} ${description} as the router's refusal`, async () => { + createMappingStub.rejects(refusal(code, description)); + const before = upnpService.routerRefusals(); - const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); - expect(result).to.include({ mapped: false, code: 501, routerWide: false }); - expect(upnpService.routerRefusals()).to.equal(before); + expect(result).to.include({ mapped: false, code, routerWide: true }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + }); + }); + + [ + [402, 'InvalidArgs'], + [716, 'WildCardNotPermittedInExtPort'], + [718, 'ConflictInMappingEntry'], + [724, 'SamePortValuesRequired'], + [725, 'OnlyPermanentLeasesSupported'], + [726, 'RemoteHostOnlySupportsWildcard'], + [727, 'ExternalPortOnlySupportsWildcard'], + [729, 'ConflictWithOtherMechanisms'], + ].forEach(([code, description]) => { + it(`should give a ${code} ${description} as the port's refusal, not the router's`, async () => { + createMappingStub.rejects(refusal(code, description)); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); }); it('should take a refusal as mapped when the router already maps the port to this node', async () => { From a98a1f2adb93279dcdd3eb790af87b4a93ba9c05 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 19:59:58 +0100 Subject: [PATCH 011/369] fix(upnp): a node whose router already maps its api port verifies on a full table Verification adds a new test mapping, which a full table refuses, so a node whose ports are all mapped shut down for want of one free entry. When the router refuses the test mapping for a reason of its own and there are no left-over test mappings to remove, the node now looks up its api port: if the router maps it to this node, enabled and port for port, UPnP works and the node runs on what it holds. Every mapping refused from then on counts toward the router DOS, which takes the node out of service until a slot frees. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 13 ++++++ tests/unit/upnpService.test.js | 70 +++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index aaa3e053b1..dca0b90c1e 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -235,6 +235,19 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { log.warn(`VerifyUPNPsupport - left-over test mappings not removed: ${error.message}`); } if (removed > 0) testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test'); + // A full table refuses every new mapping, so a node whose router already + // maps its api port to it runs on what it holds: every mapping refused from + // here on counts toward the router DOS, which takes the node out of service + // until a slot frees. + if (!testMapping.mapped) { + const apiMapping = await client.getMapping({ public: +apiport, protocol: 'TCP' }).catch(() => null); + if (mapsHere(apiMapping, +apiport)) { + log.warn(`VerifyUPNPsupport - ${testMapping.reason}, but the router maps this node's api port to it`); + mappedHere.delete(+apiport + 3); + upnpMachine = true; + return true; + } + } } if (!testMapping.mapped) { log.error(`VerifyUPNPsupport - Failed Create Mapping: ${testMapping.reason}`); diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index f64c571da4..0b30e39ec1 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -270,6 +270,11 @@ describe('upnpService tests', () => { }); it('should remove the test mappings earlier runs left, and nothing else, then map once more', async () => { + // the api port mapped here too: a test mapped on the retry is still checked and removed + natUpnp.Client.prototype.getMapping.withArgs({ public: 16127, protocol: 'TCP' }).resolves({ + ...testMapping, public: { host: '', port: 16127 }, private: { host: '192.168.1.10', port: 16127 }, + }); + const result = await upnpService.verifyUPNPsupport(); expect(result).to.equal(true); @@ -308,6 +313,71 @@ describe('upnpService tests', () => { sinon.assert.calledTwice(createMappingStub); }); + describe('and holds nothing of this node\'s to remove', () => { + // the router's entry for each port: this node's api port, or nothing + const routerMaps = (apiEntry) => { + natUpnp.Client.prototype.getMapping.reset(); + natUpnp.Client.prototype.getMapping.callsFake(async ({ public: port }) => (port === 16127 ? apiEntry : null)); + }; + const apiEntry = (overrides = {}) => ({ + ...testMapping, public: { host: '', port: 16127 }, private: { host: '192.168.1.10', port: 16127 }, description: 'Flux_Backend_API', ...overrides, + }); + + beforeEach(() => { + createMappingStub.reset(); + createMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'AddPortMapping')); + getMappingsStub.resolves([listed(31002, 'Flux_App_web')]); + }); + + it('should verify when the router already maps this node\'s api port to it', async () => { + routerMaps(apiEntry()); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(true); + expect(upnpService.isUPNP()).to.equal(true); + sinon.assert.calledWithExactly(natUpnp.Client.prototype.getMapping, { public: 16127, protocol: 'TCP' }); + sinon.assert.calledOnce(createMappingStub); + sinon.assert.notCalled(removeMappingStub); + expect(upnpService.mappedHere.has(16130)).to.equal(false); + }); + + [ + ['holds no mapping of the api port', null], + ['maps the api port to another machine', apiEntry({ local: false, private: { host: '192.168.1.11', port: 16127 } })], + ['maps the api port disabled', apiEntry({ enabled: false })], + ].forEach(([what, entry]) => { + it(`should fail when the router ${what}`, async () => { + routerMaps(entry); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + expect(upnpService.isUPNP()).to.equal(false); + }); + }); + + it('should fail on a refusal of the port\'s own, whatever the router maps', async () => { + routerMaps(apiEntry()); + createMappingStub.reset(); + createMappingStub.rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + }); + + it('should fail when the router does not answer, whatever it maps', async () => { + routerMaps(apiEntry()); + createMappingStub.reset(); + createMappingStub.rejects(new Error('timeout of 10000ms exceeded')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + }); + }); + it('should list nothing when the router does not answer', async () => { createMappingStub.onFirstCall().rejects(new Error('timeout of 10000ms exceeded')); From 8c3a38fdb4ff475f69f22397a94087c2db3f370f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 19:59:58 +0100 Subject: [PATCH 012/369] test(harness): 3001 starts a node whose own ports fill its router's table The stub gateway records the port each request names. The new case fills the table with the node's own four ports and other machines' mappings, so the test mapping finds no room and there is nothing to remove: the node maps its api port, which a node that failed verification never does, and the table is left as it was. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 2 +- .../runner/framework/external-http-control.js | 3 +- ...ehind-a-full-router-table-verifies-upnp.js | 42 +++++++++++++++++++ 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 0a76ea641e..eafa884f69 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -768,7 +768,7 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { const action = String(req.get('SOAPAction') || '').replace(/"/g, '').split('#')[1] || ''; const arg = (name) => (String(req.body).match(new RegExp(`<${name}>([^<]*)`)) || [])[1] ?? ''; const sameEntry = (m) => m.externalPort === Number(arg('NewExternalPort')) && m.protocol === arg('NewProtocol').toUpperCase(); - gateway.requests.push({ action, at: new Date().toISOString() }); + gateway.requests.push({ action, port: Number(arg('NewExternalPort')) || null, at: new Date().toISOString() }); switch (action) { case 'GetExternalIPAddress': soapReply(res, action, { NewExternalIPAddress: gateway.externalIp ?? clientAddress(req) }); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index c172b6601e..79a020205b 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -204,7 +204,8 @@ export async function enableUpnpGateway(ip, { externalIp = null, capacity = null } /** - * The gateway a node is behind: its port mappings and the actions the node sent it. + * The gateway a node is behind: its port mappings and the actions the node sent it, each with + * the external port it named (null for an action that names none). * @param {string} ip * @returns {Promise<{externalIp: string|null, mappings: object[], requests: object[]}|null>} */ diff --git a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js index ceeda39457..8337023b4a 100644 --- a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js +++ b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js @@ -78,3 +78,45 @@ describe('a node behind a full router table verifies UPnP', function () { expect(mappings.find((m) => m.externalPort === OTHERS.externalPort)).to.deep.equal(OTHERS); }); }); + +// The same router with nothing of this node's to remove: it holds the node's own four ports, +// mapped by an earlier run, and other machines' mappings up to its capacity. The test mapping +// finds no room, and the router already maps this node's api port to it, so the node runs on +// what it holds. +describe('a node whose own ports fill its router\'s table verifies UPnP', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + const OWN = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2] + .map((port, i) => mapping(port, ['Flux_Home_UI', 'Flux_Backend_API', 'Flux_Backend_API_SSL', 'Flux_Syncthing'][i])); + const OTHER_MACHINES = [32000, 32001].map((port) => mapping(port, 'Flux_App_sibling', OTHER_MACHINE)); + const FULL = [...OWN, ...OTHER_MACHINES]; + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + await enableUpnpGateway(nodeIp, { capacity: FULL.length, mappings: FULL }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('starts on its api port, its mappings and the other machines\' as they were', async function () { + this.timeout(180000); + // past the 120s a node that failed verification waits before it exits + await new Promise((resolve) => { setTimeout(resolve, 150000); }); + const { mappings, requests } = await upnpGatewayState(nodeIp); + + expect(requests.some((r) => r.action === 'AddPortMapping' && r.port === API_PORT + 3), 'canary: the node asked for its test mapping').to.equal(true); + // a node that failed verification never maps its own ports + expect(requests.some((r) => r.action === 'AddPortMapping' && r.port === API_PORT), 'the node mapped its api port').to.equal(true); + expect(mappings.map((m) => m.externalPort).sort()).to.deep.equal(FULL.map((m) => m.externalPort).sort()); + expect(requests.filter((r) => r.action === 'DeletePortMapping'), 'nothing removed').to.deep.equal([]); + }); +}); From 7b9dbedfab26087c02f53eda9c55c20d1c2f933d Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 15 Oct 2026 20:05:21 +0100 Subject: [PATCH 013/369] test(harness): 3001 changes the node's api port only once its first boot has settled The first boot writes the node's identity into userconfig.js, and a write that lands after the suite's port change puts the old port back, so FluxOS restarts on 16127. Co-Authored-By: Claude Opus 5.5 --- .../3001-a-node-behind-a-full-router-table-verifies-upnp.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js index 8337023b4a..83b9a9fca5 100644 --- a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js +++ b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js @@ -7,7 +7,7 @@ import { createTestEnv } from '../framework/test-env.js'; import { restartFluxos, setNodeApiPort } from '../framework/container.js'; import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; -import { waitFor } from '../framework/wait.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const subnet = getSubnetConfig(); @@ -48,6 +48,8 @@ describe('a node behind a full router table verifies UPnP', function () { this.timeout(240000); env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); await enableUpnpGateway(nodeIp, { capacity: CAPACITY, mappings: [...LEFT_OVER, OTHERS] }); await setNodeApiPort(env.clients[0].container, API_PORT); await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); @@ -96,6 +98,8 @@ describe('a node whose own ports fill its router\'s table verifies UPnP', functi this.timeout(240000); env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); await enableUpnpGateway(nodeIp, { capacity: FULL.length, mappings: FULL }); await setNodeApiPort(env.clients[0].container, API_PORT); await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); From f4c33290e5142e5473e9247fd7aa0e675fca54a7 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:01:27 +0100 Subject: [PATCH 014/369] fix(election): the primary of a g: app runs every g: component An app whose compose carries more than one g: component had only the first started: the election decided one component per app, and every other g: component sat created on every instance with its folder receiving. The election now decides the app. Its primary runs every g: component on one node and every other holder runs none. A node takes the app over only once each g: folder's copy is ready; a primary missing one of them starts it once that copy is. A peer holding any g: component of the app holds the app, for the start paths, the returning-primary and sending-folder decisions, and the undecided-primary check. An owner stop or start covers each g: component, and a lock taken on the node running any of them is the primary's. The seed of a g: folder is decided for the app: the node running its other g: components seeds a folder nothing has seeded, and a cold start ranks the holders on what they hold of every g: folder, so all of an app's folders are seeded on one node. A release that elects one g: component asks a peer only about the app's first, so a node holding any g: component of an app reports holding the first in /apps/heldcomponents. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 454 +++++++++++------- .../services/appManagement/appController.js | 42 +- .../services/appMonitoring/peerComponent.js | 80 +-- .../syncthingFolderStateMachine.js | 99 +++- .../appMonitoring/syncthingMonitor.js | 8 + .../src/services/appQuery/appQueryService.js | 47 +- ZelBack/src/services/utils/mountParser.js | 35 +- ...-components-run-together-on-its-primary.js | 161 +++++++ ...c-update-adds-is-started-by-the-primary.js | 136 ++++++ tests/unit/advancedWorkflows.test.js | 374 ++++++++++++++- tests/unit/appController.test.js | 65 +++ tests/unit/appQueryService.test.js | 105 +++- tests/unit/mountParser.test.js | 53 +- tests/unit/peerComponent.test.js | 99 +++- .../unit/syncthingFolderStateMachine.test.js | 126 ++++- tests/unit/syncthingMonitor.test.js | 26 + 16 files changed, 1637 insertions(+), 273 deletions(-) create mode 100644 test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js create mode 100644 test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index df5d344c68..e09033b281 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -3756,23 +3756,6 @@ function isElectedPrimaryHere(appName, localSocketAddr, now = Date.now()) { return namesAPrimary ? false : null; } -/** - * The identifier of an app's `g:` component, or null when it has none. - * - * Derived the same way masterSlaveApps derives it - the app name below v4, and - * `_` above it - because it names the same thing: the one - * component that runs on a single node at a time and writes to the volume. - * @param {object} installedApp Locally installed app record. - * @returns {string|null} - */ -function gComponentIdentifier(installedApp) { - if (installedApp.version <= 3) { - return mountParser.isGComponent(installedApp.containerData) ? installedApp.name : null; - } - const component = (installedApp.compose || []).find((c) => mountParser.isGComponent(c.containerData)); - return component ? `${component.name}_${installedApp.name}` : null; -} - /** * Whether this node should give up an app, and why. * @@ -3808,7 +3791,10 @@ async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, // by the shared ordering so every node names the same surplus const ordered = [...runningAppList].sort((a, b) => compareInstanceSeniority(b, a)); const index = ordered.findIndex((x) => socketAddressesMatch(x.ip, localSocketAddr)); - const writer = gComponentIdentifier(installedApp); + // The app's g: components: run on one node at a time, by its primary, and + // together they are what that node writes. + const writers = mountParser.gComponentIdentifiers(installedApp); + const writer = writers.join(', '); const detail = `running on ${runningAppList.length} instances (max: ${minInstances}) and this node is the newest`; // "THE NEWEST STANDS ASIDE" IS A STAND-IN FOR "THE LEAST VALUABLE COPY @@ -3824,9 +3810,8 @@ async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, // not down, and interrupting the one node serving it to tidy up the count // is a worse outcome than the count being wrong for another pass. if (index === 0) { - // eslint-disable-next-line no-await-in-loop - const runsWriter = Boolean(writer) && Boolean(deps.isComponentRunningLocally) - && await deps.isComponentRunningLocally(writer); + const runsWriter = Boolean(deps.isComponentRunningLocally) + && (await Promise.all(writers.map((each) => deps.isComponentRunningLocally(each)))).includes(true); if (!runsWriter) return { giveUp: true, reason: 'SURPLUS', detail }; // Carried out to the evacuation gate rather than returned past it, for the // reason the second-newest branch below is: a node that is also draining @@ -3849,12 +3834,11 @@ async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, // Silence, a timeout, a refusal, "not running": all mean this node does // nothing, and nothing is exactly today's behaviour. The rule can only ever // fail towards no trim, never towards two. - if (index === 1 && writer && deps.liveness) { - const appId = dockerService.getAppIdentifier(writer); + if (index === 1 && writers.length && deps.liveness) { // eslint-disable-next-line no-await-in-loop const newestState = await peerComponentState(ordered[0].ip, { - appId, - identifier: writer, + appIds: writers.map((each) => dockerService.getAppIdentifier(each)), + identifier: writers[0], appName: installedApp.name, liveness: deps.liveness, label: 'the newest copy', @@ -4881,6 +4865,162 @@ function staggerMs(places) { return places * (config.fluxapps.masterSlaveStaggerMs ?? 3 * 60 * 1000); } +/** + * An app's g: components as the election decides them, in compose order, each + * with its operator stop lock. `free` is a component the election may run here: + * not locked, or locked and started by its operator. + * @param {object} installedApp Installed app specification, decrypted. + * @returns {Promise>} + */ +async function electedMembers(installedApp) { + const identifiers = mountParser.gComponentIdentifiers(installedApp); + const stops = await Promise.all(identifiers.map((identifier) => appsRuntimeState.operatorStopState(identifier))); + return identifiers.map((identifier, index) => ({ + identifier, + appId: dockerService.getAppIdentifier(identifier), + stop: stops[index], + free: !stops[index].stopped || stops[index].startRequested, + })); +} + +/** + * Whether this node holds a g: component: it runs here, a promotion to run it is + * in progress, or the reconciler has been asked to run it. + * @param {{identifier: string}} member + * @param {string[]} runningAppsNames Identifiers running on this node. + * @returns {boolean} + */ +function holdsMemberHere(member, runningAppsNames) { + return runningAppsNames.includes(member.identifier) + || primaryRole.inTransition(member.identifier) === primaryRole.Role.PROMOTING + || appReconciler.committedIdentifiers().includes(member.identifier); +} + +/** + * The g: components whose copy here is not yet one this node may run them on. A + * copy is ready when the syncthing monitor has marked it so, or its folder + * already sends here - which covers a ready mark lost with the process. + * @param {string} appName Global app name. + * @param {Array<{identifier: string, appId: string}>} members Components to check. + * @param {Map} receiveOnlyCache The syncthing monitor's per-folder state. + * @returns {Promise} identifiers of the components not ready + */ +async function unreadyMembers(appName, members, receiveOnlyCache) { + const unmarked = members.filter((member) => !receiveOnlyCache.get(member.appId)?.restarted); + if (!unmarked.length) return []; + let folders = []; + try { + const configured = await syncthingServiceModule.getConfigFolders(); + if (Array.isArray(configured)) folders = configured; + } catch (error) { + log.error(`masterSlaveApps: Failed to check syncthing folder status for ${appName}: ${error.message}`); + } + return unmarked + .filter((member) => { + // Syncthing syncs the entire appId folder (includes all subdirectories) + const sending = folders.some((folder) => folder.path === `${appsFolder}${member.appId}` && folder.type === 'sendreceive'); + if (sending) log.info(`masterSlaveApps: app:${appName} folder ${member.appId} is already in sendreceive mode, treating as ready`); + return !sending; + }) + .map((member) => member.identifier); +} + +/** + * Begins the start of each given g: component this node may run, in compose + * order: an operator-started one through its stop lock, any other as primary. + * @param {object[]} members The components to start. + * @param {string[]} runningAppsNames Identifiers running on this node. + * @returns {Promise} Whether a start was begun for any of them. + */ +async function startMembers(members, runningAppsNames) { + let begun = false; + // eslint-disable-next-line no-restricted-syntax + for (const member of members) { + if (!member.free || runningAppsNames.includes(member.identifier)) continue; + const started = member.stop.startRequested + // eslint-disable-next-line no-await-in-loop + ? await startOperatorStarted(member.identifier, member.appId) + : startAsPrimary(member.identifier, member.appId); + begun = started || begun; + } + return begun; +} + +/** + * Starts the g: components of an app that this node, as its primary, does not + * hold yet. + * + * A node holding none of them is taking the app over, and starts once every + * copy is ready: the primary runs them all, and one started on a partial copy + * publishes it as the truth. A node already holding some is the app's writer, + * and starts each missing one once that one's own copy is ready - a component + * the spec added, or one whose start failed. + * + * Either way it asks every other holder first. One that holds any g: component + * of the app is a second primary, and starting beside it splits the app across + * two nodes; it is stood down by FDM naming one of them, and the start waits for + * that. + * @param {string} appName Global app name. + * @param {object[]} members The app's g: components. + * @param {string[]} runningAppsNames Identifiers running on this node. + * @param {Map} receiveOnlyCache The syncthing monitor's per-folder state. + * @param {object} options + * @param {() => Promise} options.othersHold The PeerComponent state of + * the other holders. + * @param {boolean} [options.namedPrimary] FDM names this node the primary. A + * holder this node cannot rule out then does not stop the start: FDM found + * nothing running from where FDM stands, and a holder that was taking the app + * over stands down for a node FDM names that has decided it holds it. + * @returns {Promise} Whether a start was begun. + */ +async function startMissingMembers(appName, members, runningAppsNames, receiveOnlyCache, { othersHold, namedPrimary = false }) { + const { identifier } = members[0]; + const missing = members.filter((member) => member.free && !holdsMemberHere(member, runningAppsNames)); + if (!missing.length) return false; + const holdsHere = members.some((member) => holdsMemberHere(member, runningAppsNames)); + const unready = await unreadyMembers(appName, missing, receiveOnlyCache); + const toStart = holdsHere ? missing.filter((member) => !unready.includes(member.identifier)) : []; + if (!holdsHere && !unready.length) toStart.push(...missing); + if (unready.length) { + fluxEventBus.count('masterSlave:decision', identifier, 'notReady'); + log.info(`masterSlaveApps: app:${appName} is ${namedPrimary ? 'registered as primary on FDM' : 'held here'} but not ready yet (syncthing not synced${members.length > 1 ? ` for ${unready.join(', ')}` : ''}), skipping start for this cycle`); + } + if (!toStart.length) return false; + const others = await othersHold(); + if (others === PeerComponent.RUNNING || (!namedPrimary && others === PeerComponent.UNKNOWN)) { + fluxEventBus.count('masterSlave:decision', identifier, 'namedButHeldElsewhere'); + log.info(`masterSlaveApps: not starting app:${appName} that ${namedPrimary ? 'FDM names this node the primary of' : 'this node holds'} - a peer ${others === PeerComponent.RUNNING ? 'holds it' : 'could not be ruled out'}`); + return false; + } + const begun = await startMembers(toStart, runningAppsNames); + if (begun) log.info(`masterSlaveApps: starting docker component:${toStart.map((member) => member.identifier).join(', ')}`); + return begun; +} + +/** + * Lifts the stop lock of every g: component of an app its operator has started. + * @param {object[]} members The app's g: components. + * @param {string} outcome What the election decided. + * @returns {Promise} + */ +async function settleOperatorStarts(members, outcome) { + // eslint-disable-next-line no-restricted-syntax + for (const member of members) { + // eslint-disable-next-line no-await-in-loop + if (member.stop.startRequested) await settleOperatorStart(member.identifier, outcome); + } +} + +/** + * Withdraws this node's seed mark from the folder of every g: component of an + * app, once the election has decided another node runs them. + * @param {object[]} members The app's g: components. + * @param {string} outcome What the election decided. + */ +function withdrawSeedMarks(members, outcome) { + members.forEach((member) => withdrawSeedMark(member.appId, outcome)); +} + /** * Manages syncthing master/slave application coordination using FDM services. * State (the busy lists, the receive-only cache) is read off globalStateParam at @@ -5016,9 +5156,6 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled.data) { let fdmOk = true; - let identifier; - let needsToBeChecked = false; - let appId; const backupSkip = globalStateParam.backupInProgress.some((backupItem) => installedApp.name === backupItem); const restoreSkip = globalStateParam.restoreInProgress.some((backupItem) => installedApp.name === backupItem); if (backupSkip || restoreSkip) { @@ -5027,42 +5164,43 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-continue continue; } - // Check all g: mode apps, not just those in cache with restarted flag - // The cache tracks sync state, but shouldn't gate primary selection - identifier = mountParser.electedComponentIdentifier(installedApp); - if (identifier) { - appId = dockerService.getAppIdentifier(identifier); - needsToBeChecked = true; - } - if (needsToBeChecked) { - // One per pass per g: component, whatever the pass decides. Every start + // Every g: component of the app. The election decides the app: its primary + // runs all of them on one node, and every other holder runs none. Checked + // for all g: mode apps, not just those in cache with restarted flag - the + // cache tracks sync state, but shouldn't gate primary selection. + // eslint-disable-next-line no-await-in-loop + const members = await electedMembers(installedApp); + if (members.length) { + // The anchor keys this app's election records and names it in the counters. + const { identifier } = members[0]; + // One per pass per g: app, whatever the pass decides. Every start // counts 'started', so passes counted from an event with no new start // is a decision not to start, taken that many times. fluxEventBus.count('masterSlave:decision', identifier, 'evaluated'); - // This node stopped the component in order to hand the app back, so it - // is not a candidate. Without this the election restores it within one - // cycle: the component is not running here, this node's own stale - // primary record is cleared below, no peer has picked it up yet, and the - // index-0 branch starts it again - every 30s, forever. Checked in the - // same place and for the same reason as operator-stopped, which is the - // other way a component this node holds is deliberately not running. - if (standingDown.has(identifier)) { - log.info(`masterSlaveApps: ${identifier} is standing down to be handed back - excluded from primary election`); + // This node stopped the app in order to hand it back, so it is not a + // candidate. Without this the election restores it within one cycle: the + // components are not running here, this node's own stale primary record + // is cleared below, no peer has picked it up yet, and the index-0 branch + // starts it again - every 30s, forever. Checked in the same place and for + // the same reason as operator-stopped, which is the other way a + // component this node holds is deliberately not running. + const handedBack = members.find((member) => standingDown.has(member.identifier)); + if (handedBack) { + log.info(`masterSlaveApps: ${handedBack.identifier} is standing down to be handed back - excluded from primary election`); // eslint-disable-next-line no-continue continue; } - // operator explicitly stopped this g: component; don't elect or act on it - // until they start it - // eslint-disable-next-line no-await-in-loop - const operatorStop = await appsRuntimeState.operatorStopState(identifier); - if (operatorStop.stopped && !operatorStop.startRequested) { + // operator explicitly stopped every g: component of the app; don't elect + // or act on it until they start one. A component stopped on its own stays + // down by its lock, and the election decides the rest. + if (!members.some((member) => member.free)) { // Outside the once-guard below on purpose: the log line reports the // state change, the counter reports every pass that honoured it, which // is what a test asserting "the election kept skipping it" needs. fluxEventBus.count('masterSlave:decision', identifier, 'operatorStopped'); if (!operatorStoppedNoted.has(identifier)) { operatorStoppedNoted.add(identifier); - log.info(`masterSlaveApps: ${identifier} is operator-stopped - excluded from primary election until it is started`); + log.info(`masterSlaveApps: ${members.map((member) => member.identifier).join(', ')} is operator-stopped - excluded from primary election until it is started`); } // eslint-disable-next-line no-continue continue; @@ -5070,14 +5208,16 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, operatorStoppedNoted.delete(identifier); // Started by its operator, and still locked: this pass decides it, and // the lock - which is what keeps this node holding the component - is - // lifted only by a decision. Starting it here, or finding it runs + // lifted only by a decision. Starting it here, or finding the app runs // elsewhere, lifts it; a pass that cannot tell leaves it for the next. - const resuming = operatorStop.startRequested; + const resumingMembers = members.filter((member) => member.stop.startRequested); + const resuming = resumingMembers.length > 0; if (resuming) { fluxEventBus.count('masterSlave:decision', identifier, 'operatorStartPending'); - log.info(`masterSlaveApps: ${identifier} was started by its operator - deciding it with the stop lock held`); + log.info(`masterSlaveApps: ${resumingMembers.map((member) => member.identifier).join(', ')} was started by its operator - deciding it with the stop lock held`); } - const start = async () => (resuming ? startOperatorStarted(identifier, appId) : startAsPrimary(identifier, appId)); + const runsHere = members.some((member) => runningAppsNames.includes(member.identifier)); + const start = () => startMembers(members, runningAppsNames); // eslint-disable-next-line no-await-in-loop await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.MASTERSLAVE_BEFORE_DECISION, identifier); // Get master IP from FDM using the new /appips endpoint @@ -5128,38 +5268,16 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, primaryElectionCheckedAt.set(identifier, Date.now()); if ((!ip)) { log.info(`masterSlaveApps: app:${installedApp.name} has currently no primary set`); - if (!runningAppsNames.includes(identifier)) { - // Check if app is ready (syncthing data is synced) before allowing it to become primary - let isReady = globalStateParam.receiveOnlySyncthingAppsCache.has(appId) && globalStateParam.receiveOnlySyncthingAppsCache.get(appId).restarted; - - // Fallback: If not in cache or not ready, check if syncthing folder is already in sendreceive mode - // This handles the case where folder is synced but cache was cleared/lost - if (!isReady) { - try { - // eslint-disable-next-line global-require - const syncthingService = require('../syncthingService'); - // eslint-disable-next-line no-await-in-loop - const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (Array.isArray(allSyncthingFolders)) { - // Syncthing syncs the entire appId folder (includes all subdirectories) - const folder = `${appsFolder}${appId}`; - // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders) { - if (syncthingFolder.path === folder && syncthingFolder.type === 'sendreceive') { - log.info(`masterSlaveApps: app:${installedApp.name} folder is already in sendreceive mode, treating as ready`); - isReady = true; - break; - } - } - } - } catch (error) { - log.error(`masterSlaveApps: Failed to check syncthing folder status for ${installedApp.name}: ${error.message}`); - } - } - - if (!isReady) { + if (!runsHere) { + // Check every g: component's copy is ready (syncthing data is + // synced) before allowing this node to become primary: the + // primary runs them all, and one started on a partial copy + // publishes it as the truth. + // eslint-disable-next-line no-await-in-loop + const unready = await unreadyMembers(installedApp.name, members.filter((member) => member.free), globalStateParam.receiveOnlySyncthingAppsCache); + if (unready.length) { fluxEventBus.count('masterSlave:decision', identifier, 'notReady'); - log.info(`masterSlaveApps: app:${installedApp.name} is not ready yet (syncthing not synced), skipping primary selection for this cycle`); + log.info(`masterSlaveApps: app:${installedApp.name} is not ready yet (syncthing not synced${members.length > 1 ? ` for ${unready.join(', ')}` : ''}), skipping primary selection for this cycle`); // eslint-disable-next-line no-continue continue; } @@ -5182,13 +5300,13 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // to clear this map. Drop it and let the normal paths decide. if (mastersRunningGSyncthingApps.has(identifier) && ipsMatch(mastersRunningGSyncthingApps.get(identifier), localSocketAddr) - && !runningAppsNames.includes(identifier)) { + && !runsHere) { mastersRunningGSyncthingApps.delete(identifier); log.info(`masterSlaveApps: cleared this node's own stale primary record for ${identifier} - it is not running here`); } - // Probe every other holder to see whether the g: component is - // already running, committed or held somewhere else. Every start + // Probe every other holder to see whether any g: component of the + // app is already running, committed or held somewhere else. Every start // path asks all of them, whatever this node's place in the order: // the order decides who goes first, not who can have started. A // node further down can hold a booking made before the one ahead @@ -5220,7 +5338,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // them together bounds the wait at one timeout regardless of peer // count. const probeCtx = { - appId, identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + appIds: members.map((member) => member.appId), identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', }; const checkPeersRunning = async () => { @@ -5231,13 +5349,13 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } const state = await componentStateOnPeers(peers, probeCtx); if (state === PeerComponent.RUNNING) { - withdrawSeedMark(appId, 'a peer holds it'); - if (resuming) await settleOperatorStart(identifier, 'a peer holds it'); + withdrawSeedMarks(members, 'a peer holds it'); + if (resuming) await settleOperatorStarts(members, 'a peer holds it'); } return state; }; - if (resuming && operatorStop.asPrimary) { + if (resumingMembers.some((member) => member.stop.asPrimary)) { // Its owner started an app this node was the primary of when it // was stopped. The start returns it here, whatever this node's // place in the order: every other node that was stopped with it @@ -5286,10 +5404,10 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-await-in-loop const previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); if (previousMasterState !== PeerComponent.NOT_RUNNING) { - if (previousMasterState === PeerComponent.RUNNING) withdrawSeedMark(appId, 'the previous primary runs it'); + if (previousMasterState === PeerComponent.RUNNING) withdrawSeedMarks(members, 'the previous primary runs it'); if (resuming && previousMasterState === PeerComponent.RUNNING) { // eslint-disable-next-line no-await-in-loop - await settleOperatorStart(identifier, 'the previous primary runs it'); + await settleOperatorStarts(members, 'the previous primary runs it'); } // Only THIS app is settled - the previous master still holds it, // so there is nothing to elect. Returning here would abandon the @@ -5376,7 +5494,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, log.info(`masterSlaveApps: holding the scheduled start of app:${installedApp.name} index: ${index} - a peer could not be ruled out`); } } else if (index > 0 && !mastersRunningGSyncthingApps.has(identifier) - && globalStateParam.receiveOnlySyncthingAppsCache.get(appId)?.designatedLeader) { + && members.some((member) => globalStateParam.receiveOnlySyncthingAppsCache.get(member.appId)?.designatedLeader)) { // The state machine's confirmed designated leader is the only // instance that can seed a newborn app: at genesis every other // instance is receiveonly with nothing to sync from, so serving @@ -5412,8 +5530,10 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // already-syncing branch and never reaches the election again. // Spent here, so it cannot take this node out of the stagger on // later primary losses. - const seedCache = globalStateParam.receiveOnlySyncthingAppsCache.get(appId); - if (seedCache) seedCache.designatedLeader = false; + members.forEach((member) => { + const seedCache = globalStateParam.receiveOnlySyncthingAppsCache.get(member.appId); + if (seedCache) seedCache.designatedLeader = false; + }); } } else if (index > 0 && !mastersRunningGSyncthingApps.has(identifier) && !timeTostartNewMasterApp.has(identifier)) { // Non-primary node with no history - schedule start based on @@ -5435,14 +5555,29 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // All other cases: don't start log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - conditions not met for primary selection`); } - } else if (!appReconciler.hasControllerOpinion(identifier)) { + } else { // Running here with no primary named: this node holds the app, and // the election leaves it running. A named primary elsewhere moves // it to stopped through the standby branch below. - // eslint-disable-next-line no-await-in-loop - if (await appReconciler.adoptControllerDesired(identifier, 'running', 'masterSlave holder, no primary named')) { - fluxEventBus.count('masterSlave:decision', identifier, 'adopted'); + // eslint-disable-next-line no-restricted-syntax + for (const member of members) { + if (runningAppsNames.includes(member.identifier) && !appReconciler.hasControllerOpinion(member.identifier)) { + // eslint-disable-next-line no-await-in-loop + if (await appReconciler.adoptControllerDesired(member.identifier, 'running', 'masterSlave holder, no primary named')) { + fluxEventBus.count('masterSlave:decision', member.identifier, 'adopted'); + } + } } + // eslint-disable-next-line no-await-in-loop + await startMissingMembers(installedApp.name, members, runningAppsNames, globalStateParam.receiveOnlySyncthingAppsCache, { + othersHold: () => componentStateOnOtherHolders( + (name) => registryManagerModule.appLocation(name), + localSocketAddr, + { + appIds: members.map((member) => member.appId), identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + }, + ), + }); } } else { // This pass read a primary off FDM. Counted rather than published: @@ -5457,7 +5592,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // Every other holder, and the node FDM names as well: what a node that // has not decided its role asks before it does. const probeCtx = { - appId, identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + appIds: members.map((member) => member.appId), identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', }; const othersHold = () => componentStateOnOtherHolders( (name) => registryManagerModule.appLocation(name), @@ -5465,89 +5600,64 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, probeCtx, { also: [ip] }, ); - const holdsHere = runningAppsNames.includes(identifier) - || primaryRole.inTransition(identifier) === primaryRole.Role.PROMOTING - || appReconciler.committedIdentifiers().includes(identifier); + const holdsHere = members.some((member) => holdsMemberHere(member, runningAppsNames)); if (!ipsMatch(localSocketAddr, ip)) { - withdrawSeedMark(appId, `FDM names ${ip} its primary`); + withdrawSeedMarks(members, `FDM names ${ip} its primary`); if (resuming) { // eslint-disable-next-line no-await-in-loop - await settleOperatorStart(identifier, `FDM names ${ip} its primary`); + await settleOperatorStarts(members, `FDM names ${ip} its primary`); } - // FDM names a node that runs the component, and also one that only + // FDM names a node that runs the app, and also one that only // holds it - by a stop lock, a promotion, or a sending folder - while // nothing runs it anywhere. This node stands down for a node that has - // decided it holds the component, or that cannot be shown not to have. + // decided it holds the app, or that cannot be shown not to have. // One that has not decided gives way to a node that has, and FDM // names this one once it runs. // eslint-disable-next-line no-await-in-loop if (holdsHere && await peerUndecided(`${extractIp(ip)}:${extractPort(ip)}`, { ...probeCtx, label: 'FDM primary' })) { - log.info(`masterSlaveApps: keeping component:${identifier} - FDM names ip:${ip}, which has not decided it holds it`); - // Stands down only the g: component on this node. Non-g siblings (e.g. a - // DB cluster component that needs all instances running) keep running. - } else if (primaryRole.standDown(identifier, appId, { running: runningAppsNames.includes(identifier) })) { - log.info(`masterSlaveApps: standing down as primary of component:${identifier} - FDM names ip:${ip}, localSocketAddr is: ${localSocketAddr}`); + log.info(`masterSlaveApps: keeping app:${installedApp.name} - FDM names ip:${ip}, which has not decided it holds it`); } else { - // A standby's folder receives and never sends: what it holds is the - // primary's, and anything written here is a local change for the - // primary's copy to overwrite. A folder that returned paused asks - // every other holder whether it runs the component, as the monitor - // does, and the node FDM names as well. - // eslint-disable-next-line no-await-in-loop - await primaryRole.holdAsStandby(identifier, appId, { othersHold }); - } - } else if (runningAppsNames.includes(identifier)) { - // The primary runs here, so its folder sends - whatever demoted it - // while the container was down. - // eslint-disable-next-line no-await-in-loop - await primaryRole.holdAsPrimary(identifier, appId); - } else if (ipsMatch(localSocketAddr, ip) && !runningAppsNames.includes(identifier)) { - // Check if app is ready (syncthing data is synced) before starting - let isReady = globalStateParam.receiveOnlySyncthingAppsCache.has(appId) && globalStateParam.receiveOnlySyncthingAppsCache.get(appId).restarted; - - // Fallback: If not in cache or not ready, check if syncthing folder is already in sendreceive mode - if (!isReady) { - try { - // eslint-disable-next-line global-require - const syncthingService = require('../syncthingService'); - // eslint-disable-next-line no-await-in-loop - const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (Array.isArray(allSyncthingFolders)) { - // Syncthing syncs the entire appId folder (includes all subdirectories) - const folder = `${appsFolder}${appId}`; - // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders) { - if (syncthingFolder.path === folder && syncthingFolder.type === 'sendreceive') { - log.info(`masterSlaveApps: app:${installedApp.name} folder is already in sendreceive mode, treating as ready`); - isReady = true; - break; - } - } + // Stands down every g: component on this node, and only those. + // Non-g siblings (e.g. a DB cluster component that needs all + // instances running) keep running. One held down by its operator + // stays as its lock keeps it. + // eslint-disable-next-line no-restricted-syntax + for (const member of members.filter((each) => each.free)) { + if (primaryRole.standDown(member.identifier, member.appId, { running: runningAppsNames.includes(member.identifier) })) { + log.info(`masterSlaveApps: standing down as primary of component:${member.identifier} - FDM names ip:${ip}, localSocketAddr is: ${localSocketAddr}`); + } else { + // A standby's folder receives and never sends: what it holds is + // the primary's, and anything written here is a local change for + // the primary's copy to overwrite. A folder that returned paused + // asks every other holder whether it runs the app, as the + // monitor does, and the node FDM names as well. + // eslint-disable-next-line no-await-in-loop + await primaryRole.holdAsStandby(member.identifier, member.appId, { othersHold }); } - } catch (error) { - log.error(`masterSlaveApps: Failed to check syncthing folder status for ${installedApp.name}: ${error.message}`); } } - - // FDM also names a node that only holds the component while - // another has taken it over and is not running it yet, so a start - // here asks every other holder first and does not start beside one - // that holds it. A holder this node cannot rule out does not stop - // it: FDM found nothing running it from where FDM stands, and a - // holder that was taking it over stands down for a node FDM names - // that has decided it holds it. - // eslint-disable-next-line no-await-in-loop - const others = isReady && !holdsHere ? await othersHold() : PeerComponent.NOT_RUNNING; - if (!isReady) { - fluxEventBus.count('masterSlave:decision', identifier, 'notReady'); - log.info(`masterSlaveApps: app:${installedApp.name} is registered as primary on FDM but not ready yet (syncthing not synced), skipping start for this cycle`); - } else if (others === PeerComponent.RUNNING) { - fluxEventBus.count('masterSlave:decision', identifier, 'namedButHeldElsewhere'); - log.info(`masterSlaveApps: not starting app:${installedApp.name} that FDM names this node the primary of - a peer holds it`); - // eslint-disable-next-line no-await-in-loop - } else if (await start()) { - log.info(`masterSlaveApps: starting docker component:${identifier}`); + } else { + // FDM names this node, so it is the app's primary. Each g: + // component that runs here keeps its folder sending - whatever + // demoted it while the container was down - and each that does + // not is started. + // eslint-disable-next-line no-restricted-syntax + for (const member of members.filter((each) => runningAppsNames.includes(each.identifier))) { + // eslint-disable-next-line no-await-in-loop + await primaryRole.holdAsPrimary(member.identifier, member.appId); } + // FDM also names a node that only holds the app while another has + // taken it over and is not running it yet, so a start here asks + // every other holder first and does not start beside one that + // holds it. A holder this node cannot rule out does not stop it: + // FDM found nothing running it from where FDM stands, and a holder + // that was taking it over stands down for a node FDM names that has + // decided it holds it. + // eslint-disable-next-line no-await-in-loop + await startMissingMembers(installedApp.name, members, runningAppsNames, globalStateParam.receiveOnlySyncthingAppsCache, { + othersHold, + namedPrimary: true, + }); } } } diff --git a/ZelBack/src/services/appManagement/appController.js b/ZelBack/src/services/appManagement/appController.js index b32bba5be1..2f9a46c7bc 100644 --- a/ZelBack/src/services/appManagement/appController.js +++ b/ZelBack/src/services/appManagement/appController.js @@ -195,11 +195,11 @@ async function operatorTargetIds(appname) { if (!ids.length) { throw new Error(`Application ${appName} was not changed: this node cannot determine its components`); } - // The component whose start the election decides, read from the spec as the - // election reads it, so the two agree on which one that is. + // The components whose start the election decides, read from the spec as the + // election reads it, so the two agree on which they are. const { inPlace: [spec] } = await appQueryService.decryptEnterpriseApps([installedRes.data[0]]); - const electedId = mountParser.electedComponentIdentifier(spec); - if (!appname.includes('_')) return { ids, appName, electedId }; + const electedIds = mountParser.gComponentIdentifiers(spec); + if (!appname.includes('_')) return { ids, appName, electedIds }; // A component is addressed by name, and a name that is not one of this app's // components addresses nothing. Taking it verbatim wrote a durable operator // lock under a component that does not exist - nothing clears one, and it holds @@ -207,11 +207,27 @@ async function operatorTargetIds(appname) { if (!ids.includes(appname)) { throw new Error(`Component ${appname} is not installed on this node`); } - return { ids: [appname], appName: appname, electedId }; + return { ids: [appname], appName: appname, electedIds }; +} + +/** + * Whether this node holds the app whose g: components these are: it runs one, + * is becoming the primary of one, or has asked the reconciler to run one. Read + * for the app and not the component, because the election decides the app - a + * component stopped on the node that runs the others was stopped on its primary. + * @param {string[]} electedIds The app's g: component identifiers. + * @returns {Promise} + */ +async function holdsElectedApp(electedIds) { + const committed = appReconciler.committedIdentifiers(); + const promoting = primaryRoleChanges.promotingIdentifiers(); + if (electedIds.some((id) => committed.includes(id) || promoting.includes(id))) return true; + const actual = await Promise.all(electedIds.map((id) => appReconciler.dockerActual(id))); + return actual.some((each) => each.running === true); } async function setAppOperatorStopped(appname, stopped, { awaitPass = false, force = false, alsoRestart = false } = {}) { - const { ids, appName, electedId } = await operatorTargetIds(appname); + const { ids, appName, electedIds } = await operatorTargetIds(appname); // Components come up in compose order and go down in the reverse of it, so a // dependency outlives what writes to it: the database stops after the server it // serves, not before it. awaitPass holds each component's pass open before the @@ -232,20 +248,16 @@ async function setAppOperatorStopped(appname, stopped, { awaitPass = false, forc let awaitsElection = false; // eslint-disable-next-line no-await-in-loop const actuated = await appReconciler.applyIntent(id, async () => { - // A start of the component the election decides, while it is locked, + // A start of a component the election decides, while it is locked, // leaves the lock for the election to lift: lifted here, the component // would be held by nothing until the election next ran, and a standby // could take it in between. See appsRuntimeState.requestOperatorStart. - awaitsElection = !stopped && id === electedId && await appsRuntimeState.requestOperatorStart(id); + const elected = electedIds.includes(id); + awaitsElection = !stopped && elected && await appsRuntimeState.requestOperatorStart(id); // Read inside the slot, so no pass moves the container between this read // and the lock. A container docker cannot read is not one known to run. - const lockOptions = stopped && id === electedId - ? { - force, - asPrimary: appReconciler.committedIdentifiers().includes(id) - || primaryRoleChanges.promotingIdentifiers().includes(id) - || (await appReconciler.dockerActual(id)).running === true, - } + const lockOptions = stopped && elected + ? { force, asPrimary: await holdsElectedApp(electedIds) } : { force }; if (!awaitsElection) await appsRuntimeState.setOperatorStopped(id, stopped, lockOptions); // Raised inside the same slot as the lock, so a pass cannot read one diff --git a/ZelBack/src/services/appMonitoring/peerComponent.js b/ZelBack/src/services/appMonitoring/peerComponent.js index 01cad8e0b3..cd9d571bb9 100644 --- a/ZelBack/src/services/appMonitoring/peerComponent.js +++ b/ZelBack/src/services/appMonitoring/peerComponent.js @@ -8,8 +8,8 @@ const { const { silenceVerdict, SilenceVerdict } = require('./peerFolderLiveness'); /** - * What this node can show about a peer's copy of a g: component. UNKNOWN is not - * a soft NOT_RUNNING: only NOT_RUNNING releases the component for a start here, + * What this node can show about a peer's copy of an app's g: components. UNKNOWN + * is not a soft NOT_RUNNING: only NOT_RUNNING releases them for a start here, * because starting is what puts a second writer on a shared volume. */ const PeerComponent = Object.freeze({ @@ -33,7 +33,11 @@ const SILENCE_REASONS = Object.freeze({ }); /** - * What this node can show one peer to be doing with a component. + * What this node can show one peer to be doing with an app's g: components. + * + * The peer holds the app when it holds ANY of them. Its primary runs them all on + * one node, so a peer holding one has the others' writes too, and starting one + * beside it splits the app across two nodes. * * The election, the surplus rule and a returning primary's stand-down ask this * question through the same code. Two implementations of "is that peer running @@ -45,7 +49,7 @@ const SILENCE_REASONS = Object.freeze({ * primary, or from the instance order the surplus rule ranks. * @param {string} peerSocketAddr The peer's socket address. * @param {object} ctx Everything the probe needs that is not the peer. - * @param {string} ctx.appId Container name for the component. + * @param {string[]} ctx.appIds Container names of the app's g: components. * @param {string} ctx.identifier `_` the election keys on. * @param {string} ctx.appName Global app name, for the log lines. * @param {object} ctx.liveness Peer folder liveness, for judging silence. @@ -54,16 +58,17 @@ const SILENCE_REASONS = Object.freeze({ * @returns {Promise} A PeerComponent state. */ async function peerComponentState(peerSocketAddr, { - appId, identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }) { // Docker reports names with a leading slash, and getAppIdentifier yields - // exactly the container name for this component. Compare whole names: a + // exactly the container name for each component. Compare whole names: a // substring test also matches a longer app whose name merely begins the same // way - myapp against myapp2, or simplexsmp against simplexsmp1 - and a false - // positive here means the component is never started at all. - const peerRunsThisComponent = (appsRunning) => appsRunning.some( - (app) => (app.Names || []).some((name) => name.replace(/^\//, '') === appId), - ); + // positive here means the app is never started at all. + const holdsAny = (names) => appIds.some((appId) => names.includes(appId)); + const peerRunsThisComponent = (appsRunning) => holdsAny(appsRunning.flatMap( + (app) => (app.Names || []).map((name) => name.replace(/^\//, '')), + )); const ipToCheck = extractIp(peerSocketAddr); const portToCheck = extractPort(peerSocketAddr); @@ -71,7 +76,7 @@ async function peerComponentState(peerSocketAddr, { const { IdentityVerdict, AnswerPurpose } = peerIdentityService; const silent = () => silentPeerState(peerSocketAddr, { - appId, identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }); // "Not running" is a clearance to start a second writer, so it counts only @@ -97,9 +102,9 @@ async function peerComponentState(peerSocketAddr, { log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} signed an answer that does not list what it holds for app:${appName}, will not start`); return PeerComponent.UNKNOWN; } - if (held.includes(appId)) { + if (holdsAny(held)) { fluxEventBus.count('masterSlave:decision', identifier, 'heldOnPeer'); - log.info(`${logPrefix}: component:${identifier} is held on peer node (${label}) at ${ipToCheck}, will not start`); + log.info(`${logPrefix}: app:${appName} is held on peer node (${label}) at ${ipToCheck}, will not start`); return PeerComponent.RUNNING; } return PeerComponent.NOT_RUNNING; @@ -134,9 +139,9 @@ async function peerComponentState(peerSocketAddr, { }); const held = heldResponse?.data?.data; if (Array.isArray(held)) { - if (held.includes(appId)) { + if (holdsAny(held)) { fluxEventBus.count('masterSlave:decision', identifier, 'heldOnPeer'); - log.info(`${logPrefix}: component:${identifier} is held on peer node (${label}) at ${ipToCheck}, will not start`); + log.info(`${logPrefix}: app:${appName} is held on peer node (${label}) at ${ipToCheck}, will not start`); return PeerComponent.RUNNING; } return PeerComponent.NOT_RUNNING; @@ -165,11 +170,11 @@ async function peerComponentState(peerSocketAddr, { log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is alive but did not list what it runs for app:${appName}, will not start`); return PeerComponent.UNKNOWN; } - // Match on the g: component identifier, not the app name: non-g siblings + // Match on the g: component identifiers, not the app name: non-g siblings // (e.g. a DB cluster component) run on every node and must not be mistaken - // for the master/slave component being active there. + // for the app's primary being there. if (peerRunsThisComponent(appsRunning)) { - log.info(`${logPrefix}: component:${identifier} is running on peer node (${label}) at ${ipToCheck}, will not start`); + log.info(`${logPrefix}: app:${appName} is running on peer node (${label}) at ${ipToCheck}, will not start`); return PeerComponent.RUNNING; } return PeerComponent.NOT_RUNNING; @@ -185,21 +190,23 @@ async function peerComponentState(peerSocketAddr, { } /** - * What a peer that did not reply at all can be shown to be doing with a - * component: free only when this node's own syncthing shows the peer's - * connection to the folder gone. + * What a peer that did not reply at all can be shown to be doing with an app's + * g: components: free only when this node's own syncthing shows the peer's + * connection gone for every one of their folders. A folder still connected is a + * component the peer may still be writing. * @param {string} peerSocketAddr The peer's socket address. * @param {object} ctx As peerComponentState. * @returns {Promise} NOT_RUNNING or UNKNOWN. */ async function silentPeerState(peerSocketAddr, { - appId, identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }) { const ipToCheck = extractIp(peerSocketAddr); - const verdict = await silenceVerdict(appId, peerSocketAddr, liveness); + const verdicts = await Promise.all(appIds.map((appId) => silenceVerdict(appId, peerSocketAddr, liveness))); + const verdict = verdicts.find((each) => each !== SilenceVerdict.GONE) ?? SilenceVerdict.GONE; fluxEventBus.count('peer:silenceVerdict', identifier, verdict); if (verdict === SilenceVerdict.GONE) { - log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appId} gone - the component is free there`); + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appIds.join(', ')} gone - the app is free there`); return PeerComponent.NOT_RUNNING; } log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent for app:${appName} and ${SILENCE_REASONS[verdict]}, will not start`); @@ -208,10 +215,10 @@ async function silentPeerState(peerSocketAddr, { /** * Whether a peer is shown, in its own signed words, not to have decided it holds - * a component: it neither runs it, nor is committed to or becoming its primary, - * nor was stopped there by its owner as its primary. It holds it at most by a - * sending folder - a primary back from a stop, still deciding whether another - * holder took over, which gives way to one that has. + * an app: of its g: components it runs none, is committed to or becoming the + * primary of none, and was stopped there by its owner as the primary of none. It + * holds them at most by sending folders - a primary back from a stop, still + * deciding whether another holder took over, which gives way to one that has. * * Anything short of a signed answer saying so is false: an unsigned or unproven * reply, a peer that cannot be reached, or one too old to say what it decided. @@ -220,7 +227,7 @@ async function silentPeerState(peerSocketAddr, { * @returns {Promise} */ async function peerUndecided(peerSocketAddr, { - appId, identifier, appName, label, logPrefix, + appIds, identifier, appName, label, logPrefix, }) { const { IdentityVerdict, AnswerPurpose } = peerIdentityService; const asked = await peerIdentityService.askSigned( @@ -235,14 +242,14 @@ async function peerUndecided(peerSocketAddr, { log.info(`${logPrefix}: peer node (${label}) at ${extractIp(peerSocketAddr)} cannot be shown to be undecided about app:${appName}`); return false; } - const undecided = !decided.includes(appId); + const undecided = !appIds.some((appId) => decided.includes(appId)); if (undecided) fluxEventBus.count('masterSlave:decision', identifier, 'peerUndecided'); return undecided; } /** - * What this node can show the other holders of a component to be doing with it, - * taken together. One peer that cannot be ruled out decides on its own: every + * What this node can show the other holders of an app to be doing with its g: + * components, taken together. One peer that cannot be ruled out decides on its own: every * other peer answering "not me" says nothing about that one. * @param {Array<{ip: string, label: string}>} peers The other holders. * @param {object} ctx As peerComponentState, without `label`. @@ -259,9 +266,10 @@ async function componentStateOnPeers(peers, ctx) { } /** - * What every other holder of a component is doing with it, read from its app - * locations: the one question a primary returning paused asks before its unsent - * changes go out or are discarded, whichever pass reaches the folder first. + * What every other holder of an app is doing with its g: components, read from + * its app locations: the one question a primary returning paused asks before its + * unsent changes go out or are discarded, whichever pass reaches the folder + * first. * @param {(appName: string) => Promise>} readLocations * @param {string} localSocketAddr This node, never asked. * @param {object} ctx As peerComponentState, without `label`. @@ -277,7 +285,7 @@ async function componentStateOnOtherHolders(readLocations, localSocketAddr, ctx, try { locations = await readLocations(ctx.appName); } catch (error) { - log.warn(`${ctx.logPrefix} - ${ctx.appId}: the other holders cannot be read: ${error.message}`); + log.warn(`${ctx.logPrefix} - ${ctx.appName}: the other holders cannot be read: ${error.message}`); return PeerComponent.UNKNOWN; } const peers = (locations || []) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 1aa2ff6506..f762d27908 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1315,9 +1315,60 @@ async function revertLocalChangesIfSynced(appId, syncStatus) { return true; } +/** + * What a candidate holds of an app's g: components, for ranking the seed of one + * of their folders: the bytes of all of them, and the newest write to any. + * + * An app whose g: components are seeded on different nodes cannot start: its + * primary runs them all, and no node holds every copy. Ranked on the whole app, + * every folder of it names the same node. A candidate missing a figure for any + * of the folders has not answered. + * @param {object|null|undefined} folderClaim What it holds of this folder. + * @param {Array} siblingClaims What it holds of each other g: folder. + * @returns {object|null|undefined} The claim to rank; this folder's alone when the app has one g: component. + */ +function electedAppClaim(folderClaim, siblingClaims) { + if (!siblingClaims.length) return folderClaim; + const parts = [folderClaim, ...siblingClaims]; + if (!parts.every(isRankableClaim)) return null; + return { + bytes: parts.reduce((total, part) => total + part.bytes, 0), + newestModified: Math.max(...parts.map((part) => part.newestModified)), + }; +} + +/** + * Which node writes an app's other g: components: this one, a peer whose folder + * for one of them sends, or none. + * + * The app's primary runs all of its g: components, so a folder of one of them + * that nothing has seeded - a component the spec has just added - is the + * writer's to seed, wherever the address order falls. Seeded anywhere else, the + * writer never holds a copy it may run it on, and the node that does is not the + * primary. + * @param {string[]} siblings Folder ids of the app's other g: components. + * @param {object[]} electionList The holders in the election. + * @param {string} localSocketAddr + * @param {object} liveness Peer folder liveness. + * @returns {Promise<{here: boolean, ip?: string}|null>} + */ +async function electedAppWriter(siblings, electionList, localSocketAddr, liveness) { + const heldHere = await Promise.all(siblings.map((sibling) => appQueryService.holdsComponent(sibling))); + if (heldHere.includes(true)) return { here: true }; + const others = electionList.filter((peer) => peer?.ip && !socketAddressesMatch(peer.ip, localSocketAddr)); + const answers = await Promise.all(others.map( + async (peer) => ({ ip: peer.ip, ...await liveness.read(peer.ip) }), + )); + const writer = answers.find((answer) => answer.reachable && answer.ready + && siblings.some((sibling) => (answer.folders || []).includes(sibling))); + return writer ? { here: false, ip: writer.ip } : null; +} + /** * Handle receive-only to send-receive transition * @param {Object} params - Parameters + * @param {string[]} [params.electedSiblings] Folder ids of the app's other g: + * components, for a g: folder; empty for any other. * @returns {Promise} Updated folder config and cache */ async function handleReceiveOnlyTransition(params) { @@ -1330,6 +1381,7 @@ async function handleReceiveOnlyTransition(params) { unsyncedSubdirs, syncthingFolder, liveness, + electedSiblings = [], } = params; const folderPath = syncthingFolder.path || `${appsFolder}${appId}/appdata`; @@ -1394,8 +1446,10 @@ async function handleReceiveOnlyTransition(params) { // expires - 125 minutes with the app down. Dropped from the list here, before the // pick, when this node can show the holder is gone rather than merely silent to it. const electionList = await holderListExcludingDead(appId, runningAppList, localSocketAddr, liveness); - // What each candidate says it holds for THIS folder, from the probe the pass has - // already paid for. This node answers for itself rather than asking itself. + // What each candidate says it holds for THIS folder - and for an app with more + // than one g: component, for all of their folders (electedAppClaim) - from the + // probe the pass has already paid for. This node answers for itself rather than + // asking itself. // // Every claim is filed under the candidate's own address as the election list // carries it, because that is the string bestHolder looks it up by, and an object @@ -1403,10 +1457,13 @@ async function handleReceiveOnlyTransition(params) { // decides where the answer comes from and nothing else - keyed from anywhere but // the list, a node stops finding its own claim the moment the two spellings of // its address differ, and the ranking degrades to the address order silently. + const ownClaim = electedAppClaim(holdings, electedSiblings.map((sibling) => globalState.folderHoldings?.get(sibling))); const claims = {}; + // Whether a peer says it holds bytes of this folder itself. + let aPeerHoldsThisFolder = false; await Promise.all(electionList.map(async (peer) => { if (socketAddressesMatch(peer.ip, localSocketAddr)) { - if (holdings) claims[peer.ip] = holdings; + if (ownClaim) claims[peer.ip] = ownClaim; return; } const answer = await liveness.read(peer.ip); @@ -1417,7 +1474,9 @@ async function handleReceiveOnlyTransition(params) { // an outcome. Deliberately untested for that reason - a test would pass whether // this condition were here or not. if (answer?.ready && answer.holding && answer.holding[appId]) { - claims[peer.ip] = answer.holding[appId]; + if ((answer.holding[appId].bytes || 0) > 0) aPeerHoldsThisFolder = true; + const claim = electedAppClaim(answer.holding[appId], electedSiblings.map((sibling) => answer.holding[sibling])); + if (claim) claims[peer.ip] = claim; } })); // A node that cannot read its OWN holdings must not outrank a peer that can show it @@ -1428,8 +1487,14 @@ async function handleReceiveOnlyTransition(params) { const someoneElseHolds = Object.entries(claims) .some(([ip, claim]) => !socketAddressesMatch(ip, localSocketAddr) && (claim?.bytes || 0) > 0); const standDownOnUnknown = ownHoldingsUnknown && someoneElseHolds; - const electedLeader = !standDownOnUnknown - && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); + // The writer of the app's other g: components seeds this folder when there is + // nothing to seed it from, and no other node does. + const appWriter = electedSiblings.length + ? await electedAppWriter(electedSiblings, electionList, localSocketAddr, liveness) + : null; + const electedLeader = appWriter + ? appWriter.here && !aPeerHasData && !aPeerHoldsThisFolder + : !standDownOnUnknown && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); // The floor holderIsGone asks of a silent holder, asked of this node before // its own win can count: a node whose peers have gone quiet is the one that // fell over, and a win it confirms in that state seeds the app on a @@ -1441,12 +1506,13 @@ async function handleReceiveOnlyTransition(params) { const isLeader = electedLeader && cache.leaderStreak >= LEADER_CONFIRM_COUNT; // Announced on the pass this node is elected, so a rival that confirms on the same // pass as it can see it - see findSeedRival. + const seedHoldings = electedSiblings.length ? ownClaim : holdings; const ownSeedClaim = { address: (runningAppList || []).find((peer) => socketAddressesMatch(peer?.ip, localSocketAddr))?.ip ?? localSocketAddr, - bytes: holdings?.bytes ?? null, - newestModified: holdings?.newestModified ?? null, + bytes: seedHoldings?.bytes ?? null, + newestModified: seedHoldings?.newestModified ?? null, }; - if (electedLeader && connected) markSeed(appId, 'intent', holdings); + if (electedLeader && connected) markSeed(appId, 'intent', seedHoldings); else globalState.seedMarks.delete(appId); // Withdrawn on every unpromoted pass, so a lost election drops the claim // and the intent behind it. It is raised again only where the promotion is @@ -1492,9 +1558,9 @@ async function handleReceiveOnlyTransition(params) { if (isLeader && !blocker?.holdsWritableCopy) { // Marked before the peers are read, so a rival deciding at the same moment // sees this node. Every way out below that does not seed goes back to intent. - markSeed(appId, 'deciding', holdings); + markSeed(appId, 'deciding', seedHoldings); const notSeeding = () => { - markSeed(appId, 'intent', holdings); + markSeed(appId, 'intent', seedHoldings); decideType(syncthingFolder, containerDataFlags, 'receiveonly'); return { syncthingFolder, cache }; }; @@ -1560,7 +1626,7 @@ async function handleReceiveOnlyTransition(params) { log.info(`handleReceiveOnlyTransition - ${appId}: deciding to seed took ${decisionMs}ms, longer than a pass (${SEED_DECISION_BUDGET_MS}ms); deciding again on the next pass`); return notSeeding(); } - markSeed(appId, 'decided', holdings); + markSeed(appId, 'decided', seedHoldings); fluxEventBus.count('syncthing:seedDecision', appId, 'seeded'); // Every gate passed - but deciding the promotion is not applying it. The @@ -1605,7 +1671,8 @@ async function handleReceiveOnlyTransition(params) { const reason = `candidates=${electionList.length} self=${selfInElection} ` + `elected=${electedLeader} connected=${connected} ` + `streak=${cache.leaderStreak}/${LEADER_CONFIRM_COUNT} ` - + `peerHasData=${aPeerHasData}`; + + `peerHasData=${aPeerHasData}` + + (electedSiblings.length ? ` appWriter=${appWriter ? (appWriter.here ? 'here' : appWriter.ip) : 'none'}` : ''); if (reason !== cache.lastNotPromotedReason) { log.info(`handleReceiveOnlyTransition - ${appId} not promoted: ${reason}`); cache.lastNotPromotedReason = reason; @@ -1824,6 +1891,7 @@ async function manageFolderSyncState(params) { syncthingFolder, installedAppName, liveness, + electedSiblings = [], } = params; // One per pass per folder, whatever the pass decides: a harness suite counts @@ -1846,8 +1914,10 @@ async function manageFolderSyncState(params) { // its process, and what it holds must not go out until the election says // it is primary again. log.info(`manageFolderSyncState - ${appId} is sendreceive and not held here, demoting until the election decides`); + // Every g: component of the app is asked about: a holder running any of + // them is its primary, and this folder's changes are not to go out beside it. const othersHold = () => componentStateOnOtherHolders(readAppLocation, localSocketAddr, { - appId, identifier, appName: installedAppName, liveness, logPrefix: 'manageFolderSyncState', + appIds: [appId, ...electedSiblings], identifier, appName: installedAppName, liveness, logPrefix: 'manageFolderSyncState', }); await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_STANDBY_HOLD, appId); await primaryRole.holdAsStandby(identifier, appId, { othersHold }); @@ -1905,6 +1975,7 @@ async function manageFolderSyncState(params) { unsyncedSubdirs, syncthingFolder, liveness, + electedSiblings, }); return result; } diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 3bedcac507..8052b1858b 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -268,6 +268,8 @@ async function appLocation(appName) { * This function handles both legacy apps (version <= 3) and newer apps (version > 3) * * @param {Object} params - Parameters object + * @param {string[]} [params.electedFolderIds] Folder ids of the app's g: + * components, which its primary runs together. * @returns {Promise} */ async function processContainerData(params) { @@ -286,6 +288,7 @@ async function processContainerData(params) { foldersConfiguration, newFoldersConfiguration, liveness, + electedFolderIds = [], } = params; const containersData = containerData.split('|'); @@ -365,6 +368,7 @@ async function processContainerData(params) { syncthingFolder, installedAppName, liveness, + electedSiblings: electedFolderIds.includes(appId) ? electedFolderIds.filter((folderId) => folderId !== appId) : [], }); // Update cache if provided @@ -834,6 +838,9 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { }); } else { // Newer app (version > 3) - compose with multiple components + // The folders of the app's g: components, which its primary runs together. + const electedFolderIds = mountParser.gComponentIdentifiers(installedApp) + .map((electedIdentifier) => dockerService.getAppIdentifier(electedIdentifier)); // eslint-disable-next-line no-restricted-syntax for (const installedComponent of installedApp.compose) { const identifier = `${installedComponent.name}_${installedApp.name}`; @@ -844,6 +851,7 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { containerData: installedComponent.containerData, identifier, installedAppName: installedApp.name, + electedFolderIds, }); } } diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index c7212c1f1d..9497f5ee5d 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -6,6 +6,7 @@ const dockerService = require('../dockerService'); const registryManager = require('../appDatabase/registryManager'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); const appConstants = require('../utils/appConstants'); +const mountParser = require('../utils/mountParser'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const fluxCaching = require('../utils/cacheManager'); @@ -381,14 +382,50 @@ async function sendingFolderIds() { } } +/** + * The container names of the g: components of each installed app that has more + * than one, anchor first (mountParser.anchorComponentIdentifier). + * + * Throws when the installed apps cannot be read. + * @returns {Promise} + */ +async function multiComponentElectedApps() { + const installed = await installedApps(); + if (installed.status !== 'success') throw new Error('the installed apps cannot be read'); + const { inPlace } = await decryptEnterpriseApps(installed.data); + return inPlace + .map((app) => mountParser.gComponentIdentifiers(app).map((identifier) => dockerService.getAppIdentifier(identifier))) + .filter((names) => names.length > 1); +} + +/** + * A holdings list with the anchor of every app it holds any g: component of. + * + * A peer on a release that elects one g: component asks only about the anchor, + * so to it a node holding another of the app's g: components - one whose + * anchor has not started yet, has stopped on its own, or failed to start - + * holds nothing, and it starts the app beside it. + * @param {string[]} holding Container names. + * @param {string[][]} apps As multiComponentElectedApps. + * @returns {string[]} + */ +function withAnchors(holding, apps) { + const names = new Set(holding); + apps.forEach((members) => { + if (members.some((name) => names.has(name))) names.add(members[0]); + }); + return [...names]; +} + /** * What this node holds, both ways a peer reads it: `decided` is its own account * (ownHoldings), and `held` adds every component whose folder sends here. A * component in `held` and not in `decided` has a sending folder and nothing * behind it yet: a primary back from a stop, still deciding whether another - * holder took over. + * holder took over. Each list carries the anchor of every app it holds a g: + * component of (withAnchors). * - * Throws when the lock store cannot be read. + * Throws when the lock store or the installed apps cannot be read. * @returns {Promise<{decided: string[], held: string[]}|null>} null while * syncthing cannot say which folders send here */ @@ -396,7 +433,11 @@ async function holdingsAccount() { const sending = await sendingFolderIds(); if (sending === null) return null; const decided = await ownHoldings(); - return { decided, held: [...new Set([...decided, ...sending])] }; + const apps = await multiComponentElectedApps(); + return { + decided: withAnchors(decided, apps), + held: withAnchors([...decided, ...sending], apps), + }; } /** diff --git a/ZelBack/src/services/utils/mountParser.js b/ZelBack/src/services/utils/mountParser.js index b6ea7b4752..215e1062a9 100644 --- a/ZelBack/src/services/utils/mountParser.js +++ b/ZelBack/src/services/utils/mountParser.js @@ -530,17 +530,33 @@ function isGComponent(containerData) { } /** - * The component of an app whose start the primary election decides: its first - * g: component, as `_` (the app name for a v1-3 app), or null - * when it has none. The election decides one component per app, this one. + * The components of an app whose start the primary election decides: every g: + * component, in compose order, as `_` (the app name for a v1-3 + * app). Empty when it has none. The election decides the app, and its primary + * runs all of them. + * @param {object} appSpec - Installed app specification, decrypted. + * @returns {string[]} + */ +function gComponentIdentifiers(appSpec) { + if (!appSpec) return []; + if (appSpec.version <= 3) return isGComponent(appSpec.containerData) ? [appSpec.name] : []; + return (appSpec.compose || []) + .filter((comp) => isGComponent(comp.containerData)) + .map((comp) => `${comp.name}_${appSpec.name}`); +} + +/** + * An app's anchor: its first g: component, or null when it has none. + * + * A FluxOS release that predates electing every g: component knows only this + * one, and asks a peer whether it holds the app by asking about it - so a node + * holding any g: component of the app reports holding the anchor + * (appQueryService.heldComponents). * @param {object} appSpec - Installed app specification, decrypted. * @returns {string|null} */ -function electedComponentIdentifier(appSpec) { - if (!appSpec) return null; - if (appSpec.version <= 3) return isGComponent(appSpec.containerData) ? appSpec.name : null; - const component = (appSpec.compose || []).find((comp) => isGComponent(comp.containerData)); - return component ? `${component.name}_${appSpec.name}` : null; +function anchorComponentIdentifier(appSpec) { + return gComponentIdentifiers(appSpec)[0] ?? null; } /** True iff the component's primary mount is any sync mount (g:/r:/s:). */ @@ -574,7 +590,8 @@ module.exports = { hasFlag, getComponentSyncMode, isGComponent, - electedComponentIdentifier, + gComponentIdentifiers, + anchorComponentIdentifier, isSyncedComponent, validateMountPath, validateSubdirOrFilename, diff --git a/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js b/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js new file mode 100644 index 0000000000..f6c0745ac3 --- /dev/null +++ b/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js @@ -0,0 +1,161 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { + execInContainer, listAppContainers, crashFluxos, releaseFluxos, +} from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, +} from '../framework/syncthing-real.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, on real syncthing daemons. +// +// The election decides the app, not each component: its primary runs every g: +// component on one node, and every other holder runs none. A node running one of +// them and not the other is the app split across two nodes - the load balancer +// sends all of its ports to one of them, and the components cannot reach each +// other over the app's network - so every wait below holds that no two nodes run +// any of its g: components at once. + +const APP_UID = 1000; + +describe('an app\'s g: components run together on its primary', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2etwin${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const folderOf = (component) => `flux${component}_${appName}`; + // Placed in this order, so the election ranks them in it; a third node holds + // nothing and keeps the ring whole. + const FIRST = 0; + const SECOND = 1; + const HOLDERS = [FIRST, SECOND]; + + const client = (i) => env.clients[i]; + // The app's g: components running on a node, in compose order. + const runningOn = async (i) => { + const containers = await listAppContainers(client(i).container); + return COMPONENTS.filter((component) => containers.some( + (c) => c.name === `flux${component}_${appName}` && c.status?.startsWith('Up'), + )); + }; + // Which holders run any of the app's g: components; asserted on every poll. + const oneHolderAtMost = async () => { + const running = await Promise.all(HOLDERS.map(runningOn)); + const holders = HOLDERS.filter((_, k) => running[k].length); + expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); + return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); + }; + const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (component) => (await getFolderConfig(client(i), folderOf(component)))?.type)); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'twin', { user: APP_UID }); + const app = await buildSeedableApp({ + env, + name: appName, + instances: HOLDERS.length, + compose: COMPONENTS.map((component) => ({ + name: component, + description: `${component} of a two-writer app`, + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [`WRITE_FILE=/appdata/${component}.txt`, `WRITE_CONTENT=${component}`], + commands: [], + containerPorts: [80], + containerData: 'g:/appdata', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + })), + }); + + await installOnNodes(env, app, [FIRST]); + await waitHolding(async () => (await oneHolderAtMost())[FIRST].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + await installOnNodes(env, app, [SECOND]); + await Promise.all(COMPONENTS.map((component) => waitFor(() => isFolderSynced(client(SECOND), folderOf(component)), { + timeout: 300000, interval: 3000, label: `the second holder has the primary's ${component} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs every g: component on the primary, and none on the standby', async function () { + this.timeout(180000); + // Held over several election passes, not read once. + const deadline = Date.now() + 60000; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + const running = await oneHolderAtMost(); + expect(running[FIRST], 'the primary').to.deep.equal(COMPONENTS); + expect(running[SECOND], 'the standby').to.deep.equal([]); + // eslint-disable-next-line no-await-in-loop + await sleepUnlessInfraDead(5000); + } + expect(await folderTypes(FIRST), 'the primary\'s folders').to.deep.equal(['sendreceive', 'sendreceive']); + expect(await folderTypes(SECOND), 'the standby\'s folders').to.deep.equal(['receiveonly', 'receiveonly']); + }); + + it('moves every g: component together when the primary dies', async function () { + this.timeout(420000); + // The whole machine: the process, the containers and the daemon. + await crashFluxos(client(FIRST).container, { hold: true }); + await Promise.all(COMPONENTS.map((component) => execInContainer(client(FIRST).container, `docker kill ${folderOf(component)}`))); + await stopDaemon(client(FIRST)); + + await waitHolding(async () => (await oneHolderAtMost())[SECOND].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the second holder runs every g: component', + }); + await waitFor(async () => (await folderTypes(SECOND)).every((type) => type === 'sendreceive'), { + timeout: 120000, interval: 2000, label: 'every folder of the new primary sends', + }); + }); + + it('brings the old primary back as a standby of the whole app', async function () { + this.timeout(420000); + await startDaemon(client(FIRST), { paused: true }); + await releaseFluxos(client(FIRST).container); + + await waitFor(async () => (await folderTypes(FIRST)).every((type) => type === 'receiveonly'), { + timeout: 300000, interval: 3000, label: 'every folder of the old primary receives', + }); + const deadline = Date.now() + 60000; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + const running = await oneHolderAtMost(); + expect(running[FIRST], 'the old primary').to.deep.equal([]); + expect(running[SECOND], 'the primary').to.deep.equal(COMPONENTS); + // eslint-disable-next-line no-await-in-loop + await sleepUnlessInfraDead(5000); + } + }); +}); diff --git a/test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js b/test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js new file mode 100644 index 0000000000..9e803a6a97 --- /dev/null +++ b/test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js @@ -0,0 +1,136 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { listAppContainers } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A specification update adds a second single-writer (g:) component to a running +// app, on real syncthing daemons. +// +// Its folder is new on every holder and holds nothing anywhere, and the app's +// primary is the holder whose address sorts LAST. Seeded by the address order the +// new folder would be the standby's, and the primary - which runs every g: +// component of the app - would never hold a copy it may start the new one on. The +// primary seeds it and starts it, the standby never runs it, and at no point does +// any g: component of the app run on two nodes. +// +// DRIVEN BY BLOCKS: the reinstall pass fires on +// `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). + +const APP_UID = 1000; + +describe('a g: component a spec update adds is started by the primary', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eaddg${Date.now()}`; + const folderOf = (component) => `flux${component}_${appName}`; + // The standby sorts first by address; the primary is placed first, so it runs + // the app, and sorts last. + const STANDBY = 0; + const PRIMARY = 1; + const HOLDERS = [STANDBY, PRIMARY]; + let app; + + const client = (i) => env.clients[i]; + const component = (name) => ({ + name, + description: `${name} of an app that gains a writer`, + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], + commands: [], + containerPorts: [80], + containerData: 'g:/appdata', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }); + const runningOn = async (i) => { + const containers = await listAppContainers(client(i).container); + return ['db', 'web'].filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); + }; + // Which holders run any of the app's g: components; asserted on every poll. + const oneHolderAtMost = async () => { + const running = await Promise.all(HOLDERS.map(runningOn)); + const holders = HOLDERS.filter((_, k) => running[k].length); + expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); + return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); + }; + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + expect(`${client(STANDBY).ip}:16127` < `${client(PRIMARY).ip}:16127`, `fixture: the standby (${client(STANDBY).ip}) sorts before the primary (${client(PRIMARY).ip}) by address`).to.equal(true); + + await pushTestApp(appName, 'v1', 'addg', { user: APP_UID }); + app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: [component('db')], + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('seeds the new component\'s folder on the primary and starts it there, never on the standby', async function () { + // A backstop: the block drive bounds this test (suite 85). + this.timeout(1800000); + const updated = await buildSeedableUpdate(app, (spec) => { + spec.compose.push(component('web')); + }); + await seedSpecUpdate(env, updated, HOLDERS); + await stopTicker(); + + let breach = null; + await driveUntil(client(PRIMARY), async () => { + try { + const running = await oneHolderAtMost(); + return running[PRIMARY].length === 2; + } catch (error) { + if (error?.name !== 'AssertionError') throw error; + breach = error; + return true; + } + }, { blocks: 216, label: 'the primary runs db and web' }); + if (breach) throw breach; + + expect((await getFolderConfig(client(PRIMARY), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('web')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s web data', + }); + expect((await getFolderConfig(client(STANDBY), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + }); +}); diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index cca81e9cb7..aa2babcc11 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -22,6 +22,7 @@ const primaryRoleChanges = require('../../ZelBack/src/services/appLifecycle/prim const { appsFolder } = require('../../ZelBack/src/services/utils/appConstants'); const syncthingFolderStateMachine = require('../../ZelBack/src/services/appMonitoring/syncthingFolderStateMachine'); const { RECONNECT_GRACE_MS } = require('../../ZelBack/src/services/appMonitoring/peerFolderLiveness'); +const { PeerComponent } = require('../../ZelBack/src/services/appMonitoring/peerComponent'); const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../../ZelBack/src/services/utils/addressMoveEvents'); const appInstaller = require('../../ZelBack/src/services/appLifecycle/appInstaller'); @@ -1731,6 +1732,321 @@ describe('advancedWorkflows tests', () => { }); }); + // An app with more than one g: component is elected as a whole: its primary + // runs every one of them on one node, and every other holder runs none. + describe('an app with more than one g: component', () => { + const APP = 'twinapp'; + const DB = `db_${APP}`; + const WEB = `web_${APP}`; + const PEER = '192.168.1.90:16127'; + const namesThisNode = { data: { status: 'success', data: { ips: ['192.168.1.5'] } } }; + const namesPeer = { data: { status: 'success', data: { ips: ['192.168.1.90'] } } }; + let promote; + let standDown; + let holdAsStandby; + let holdAsPrimary; + let count; + let logInfo; + + // db and web are elected; cache runs on every instance and is never the + // election's. `running` and `ready` name components by their identifier. + const twinFixture = ({ running = [], ready = [DB, WEB], peers = [PEER] } = {}) => { + dockerServiceStub.callsFake((name) => `flux${name}`); + const installedApps = sinon.stub().resolves({ + status: 'success', + data: [{ + name: APP, + version: 8, + compose: [ + { name: 'db', containerData: 'g:/data' }, + { name: 'cache', containerData: '/cache' }, + { name: 'web', containerData: 'g:/srv' }, + ], + }], + }); + const listRunningApps = sinon.stub().resolves({ + status: 'success', + data: running.map((identifier) => ({ Names: [`/flux${identifier}`] })), + }); + ready.forEach((identifier) => globalState.receiveOnlySyncthingAppsCache.set(`flux${identifier}`, { restarted: true })); + fluxNetworkHelperStub.resolves('192.168.1.5:16127'); + registryManagerStub.resolves([ + { name: APP, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }, + ...peers.map((ip, n) => ({ name: APP, ip, runningSince: `2026-01-01T00:0${n + 1}:00.000Z` })), + ]); + // no folder sends here, so a copy is ready only when its mark says so + syncthingServiceStub.resolves([]); + let delayCalls = 0; + serviceHelperDelayStub.resetBehavior(); + serviceHelperDelayStub.callsFake(async () => { + delayCalls += 1; + if (delayCalls === 1) { + globalState.installationInProgress = true; + return undefined; + } + return new Promise(() => {}); + }); + return async () => { + delayCalls = 0; + globalState.installationInProgress = false; + await advancedWorkflows.masterSlaveApps(globalState, installedApps, listRunningApps, https); + }; + }; + const peersHold = (held) => { + axiosGetStub.resetBehavior(); + axiosGetStub.callsFake(peerAnswers({ held })); + }; + const promoted = () => promote.args.map(([identifier]) => identifier); + + beforeEach(() => { + promote = sinon.stub(primaryRole, 'promote').returns(true); + standDown = sinon.stub(primaryRole, 'standDown').returns(true); + holdAsStandby = sinon.stub(primaryRole, 'holdAsStandby').resolves(); + holdAsPrimary = sinon.stub(primaryRole, 'holdAsPrimary').resolves(); + count = sinon.stub(fluxEventBus, 'count'); + logInfo = sinon.stub(log, 'info'); + }); + + describe('with no primary named', () => { + beforeEach(() => { + serviceHelperStub.resolves(fdmNoPrimary()); + }); + + it('starts every g: component, in compose order, and no other', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture(); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([DB, WEB]); + }); + + it('starts none while one component\'s copy is not ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ ready: [DB] }); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + expect(linesMatching(logInfo, `not ready yet (syncthing not synced for ${WEB})`)).to.have.lengthOf(1); + }); + + // A peer running only the later component is the app's primary: starting + // the anchor here would split the app across two nodes. + it('starts none when a peer holds only a later g: component', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture(); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + + it('starts the component its operator did not stop, and leaves the stopped one down', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').callsFake(async (identifier) => (identifier === WEB ? LOCKED : UNLOCKED)); + const runPass = twinFixture(); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([DB]); + }); + + it('leaves the app out of the election while its operator has stopped every g: component', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(LOCKED); + const runPass = twinFixture(); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(count, 'masterSlave:decision', DB, 'operatorStopped'); + }); + + describe('on a node that already runs one of them', () => { + it('starts the other once its copy is ready and no peer holds the app', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([WEB]); + }); + + it('does not start the other beside a peer that holds the app', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + + // Canary for the test above: the same pass with the peer unreadable is + // not a clearance either, while FDM names nobody. + it('does not start the other beside a peer it cannot rule out', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + }); + }); + + describe('FDM naming this node', () => { + beforeEach(() => { + serviceHelperStub.resolves(namesThisNode); + }); + + it('keeps the running component sending and starts the one the spec added once its copy is ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + peersHold([]); + + await runPass(); + + sinon.assert.calledOnceWithExactly(holdAsPrimary, DB, `flux${DB}`); + expect(promoted()).to.deep.equal([WEB]); + }); + + it('waits for the missing component\'s copy to be ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB], ready: [DB] }); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(count, 'masterSlave:decision', DB, 'notReady'); + }); + + it('takes the app over only once every copy is ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ ready: [WEB] }); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + + it('does not take the app over while another holder holds any of it', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture(); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(count, 'masterSlave:decision', DB, 'namedButHeldElsewhere'); + }); + }); + + describe('FDM naming another node', () => { + beforeEach(() => { + serviceHelperStub.resolves(namesPeer); + }); + + it('stands down every g: component it runs', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB, WEB] }); + + await runPass(); + + expect(standDown.args.map(([identifier]) => identifier)).to.deep.equal([DB, WEB]); + sinon.assert.notCalled(promote); + }); + + // A component running here beside one that is not still makes this node + // a holder of the app, and stands down with it. + it('stands down a component that runs here even when the other does not', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + standDown.callsFake((identifier, appId, { running }) => running); + const runPass = twinFixture({ running: [WEB] }); + + await runPass(); + + sinon.assert.calledWith(standDown, WEB, `flux${WEB}`, { running: true }); + sinon.assert.calledOnceWithExactly(holdAsStandby, DB, `flux${DB}`, { othersHold: sinon.match.func }); + }); + + it('leaves a component its operator stopped as its lock keeps it', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').callsFake(async (identifier) => (identifier === WEB ? LOCKED : UNLOCKED)); + const runPass = twinFixture({ running: [DB] }); + + await runPass(); + + expect(standDown.args.map(([identifier]) => identifier)).to.deep.equal([DB]); + sinon.assert.notCalled(holdAsStandby); + }); + + // The question a standby's paused folder asks, and the one the named node + // is asked whether it has decided: both about the whole app. + it('asks the other holders about every g: component of the app', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + standDown.returns(false); + const runPass = twinFixture(); + peersHold([`flux${WEB}`]); + + await runPass(); + const { othersHold } = holdAsStandby.firstCall.args[2]; + + expect(await othersHold()).to.equal(PeerComponent.RUNNING); + }); + }); + + describe('started by its operator after a stop on this node as its primary', () => { + beforeEach(() => { + serviceHelperStub.resolves(fdmNoPrimary()); + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(START_REQUESTED_AS_PRIMARY); + sinon.stub(appsRuntimeState, 'releaseOperatorStart').resolves(true); + sinon.stub(appReconciler, 'applyIntent').callsFake(async (id, mutate) => { + await mutate(); + return true; + }); + promote.callsFake((identifier, appId) => { + primaryRoleChanges.set(identifier, { state: 'promoting', appId }); + return true; + }); + }); + + afterEach(() => { + primaryRoleChanges.promotingIdentifiers().forEach((identifier) => primaryRoleChanges.end(identifier, primaryRoleChanges.get(identifier))); + }); + + it('resumes every g: component and lifts each one\'s lock', async () => { + const runPass = twinFixture({ peers: [PEER] }); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([DB, WEB]); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, DB); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, WEB); + }); + + // A peer that took the app over decides it for every component: each lock + // is lifted, and none of them starts here. + it('lifts each one\'s lock without starting it when a peer holds the app', async () => { + const runPass = twinFixture({ peers: [PEER] }); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, DB); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, WEB); + }); + }); + }); + describe('isElectedPrimaryHere answers in three states', () => { // Finding 25 of the review: the true path had no coverage at all, and the // false path could not be told apart from "the election never ran". Each @@ -2458,7 +2774,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); }); @@ -2491,7 +2807,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); }); @@ -2530,7 +2846,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); }); @@ -2778,7 +3094,7 @@ describe('advancedWorkflows tests', () => { await runPass(); expect(linesMatching(logInfo, 'alive, and cannot be ruled out')).to.have.lengthOf(1); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(0); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(0); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); @@ -3321,7 +3637,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, `component:${appName} is held on peer node (index 2)`)).to.have.lengthOf(1); + expect(linesMatching(logInfo, `app:${appName} is held on peer node (index 2)`)).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); @@ -3607,7 +3923,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, `component:${first} is held on peer node (previous primary)`)).to.have.lengthOf(1); + expect(linesMatching(logInfo, `app:${first} is held on peer node (previous primary)`)).to.have.lengthOf(1); // the assertion that discriminates: the pass carried on to the second app expect(linesMatching(logInfo, `starting docker component:${second}`)).to.have.lengthOf(1); }); @@ -7322,6 +7638,52 @@ describe('giving up an app: one pass, two reasons, one safety gate', function () expect(decision.reason).to.equal('SURPLUS'); }); + // An app with more than one g: component is written by the node that runs + // any of them: its primary runs them together. + describe('an app with more than one g: component', () => { + const twoWriters = { + name: 'a', instances: 2, version: 8, compose: [{ name: 'w', containerData: 'g:/data' }, { name: 'v', containerData: 'g:/srv' }], + }; + const runsOnly = (identifier) => sinon.stub().callsFake(async (each) => each === identifier); + + it('leaves the newest copy alone when it runs only a later g: component', async () => { + const decision = await advancedWorkflows.reasonToGiveUpApp( + twoWriters, + locations('5.6.7.8:16127', '9.9.9.9:16127', LOCAL), + LOCAL, + { isComponentRunningLocally: runsOnly('v_a'), liveness: {} }, + ); + + expect(decision.giveUp).to.equal(false); + expect(decision.reason).to.equal('SURPLUS'); + }); + + // Canary for the test above: the same newest copy running none of them trims. + it('trims the newest copy when it runs none of them', async () => { + const decision = await advancedWorkflows.reasonToGiveUpApp( + twoWriters, + locations('5.6.7.8:16127', '9.9.9.9:16127', LOCAL), + LOCAL, + { isComponentRunningLocally: runsOnly('other_a'), liveness: {} }, + ); + + expect(decision.giveUp).to.equal(true); + }); + + it('trims the second-newest once the newest confirms it holds a later g: component', async () => { + newestSays({ held: [dockerServiceModule.getAppIdentifier('v_a')] }); + + const decision = await advancedWorkflows.reasonToGiveUpApp( + twoWriters, + locations('5.6.7.8:16127', LOCAL, '9.9.9.9:16127'), + LOCAL, + { isComponentRunningLocally: sinon.stub().resolves(false), liveness: {} }, + ); + + expect(decision.giveUp).to.equal(true); + }); + }); + it('trims the second-newest once the newest confirms it holds the writer', async () => { newestSays({ held: [writerAppId] }); diff --git a/tests/unit/appController.test.js b/tests/unit/appController.test.js index fe69e35582..e172305265 100644 --- a/tests/unit/appController.test.js +++ b/tests/unit/appController.test.js @@ -361,6 +361,71 @@ describe('appController tests', () => { }); }); + // An app with more than one g: component is elected as a whole, so its + // operator commands defer to the election for each of them, and a lock taken + // on the node running any of them is the primary's. + describe('an app with more than one g: component', () => { + const twinApp = { + name: 'Twin', + version: 4, + compose: [{ name: 'db', containerData: 'g:/data' }, { name: 'cache', containerData: '/c' }, { name: 'web', containerData: 'g:/srv' }], + }; + let requestOperatorStart; + + beforeEach(() => { + verificationHelperStub.resolves(true); + requestOperatorStart = sinon.stub(appsRuntimeState, 'requestOperatorStart').resolves(true); + sinon.stub(fluxEventBus, 'publish'); + }); + + it('leaves every g: component\'s lock for the election to lift', async () => { + stubInstalledApp(twinApp); + + await appController.appStart({ params: { appname: 'Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + expect(requestOperatorStart.args.map(([id]) => id)).to.deep.equal(['db_Twin', 'web_Twin']); + expect(appsRuntimeState.setOperatorStopped.args.map(([id]) => id)).to.deep.equal(['cache_Twin']); + }); + + it('takes each g: component\'s lock as the primary\'s when only one of them runs here', async () => { + stubInstalledApp(twinApp); + sinon.stub(appReconciler, 'clearControllerDesired'); + sinon.stub(appReconciler, 'committedIdentifiers').returns([]); + appReconciler.dockerActual.callsFake(async (id) => ({ reachable: true, exists: true, running: id === 'web_Twin' })); + + await appController.appStop({ params: { appname: 'Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + const options = Object.fromEntries(appsRuntimeState.setOperatorStopped.args.map(([id, , opts]) => [id, opts])); + expect(options).to.deep.equal({ + db_Twin: { force: false, asPrimary: true }, cache_Twin: { force: false }, web_Twin: { force: false, asPrimary: true }, + }); + }); + + it('takes a single g: component\'s lock as the primary\'s on the node running the other', async () => { + stubInstalledApp(twinApp); + sinon.stub(appReconciler, 'clearControllerDesired'); + sinon.stub(appReconciler, 'committedIdentifiers').returns([]); + appReconciler.dockerActual.callsFake(async (id) => ({ reachable: true, exists: true, running: id === 'db_Twin' })); + + await appController.appStop({ params: { appname: 'web_Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + sinon.assert.calledOnceWithExactly(appsRuntimeState.setOperatorStopped, 'web_Twin', true, { force: false, asPrimary: true }); + }); + + // Canary for the test above: on a node holding none of the app the same + // stop is not the primary's. + it('takes it as not the primary\'s on a node running none of them', async () => { + stubInstalledApp(twinApp); + sinon.stub(appReconciler, 'clearControllerDesired'); + sinon.stub(appReconciler, 'committedIdentifiers').returns([]); + appReconciler.dockerActual.resolves({ reachable: true, exists: true, running: false }); + + await appController.appStop({ params: { appname: 'web_Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + sinon.assert.calledOnceWithExactly(appsRuntimeState.setOperatorStopped, 'web_Twin', true, { force: false, asPrimary: false }); + }); + }); + it('should report an unreachable docker rather than claiming a start', async () => { verificationHelperStub.resolves(true); stubInstalledApp({ diff --git a/tests/unit/appQueryService.test.js b/tests/unit/appQueryService.test.js index 8bb79db4d1..d3a4b1f83f 100644 --- a/tests/unit/appQueryService.test.js +++ b/tests/unit/appQueryService.test.js @@ -601,11 +601,32 @@ describe('appQueryService tests', () => { }); }); + // The apps installed on this node, as the local database answers them. + const installedAppsAre = (apps) => { + dbHelperStub.databaseConnection.returns({ db: () => ({}) }); + dbHelperStub.findInDatabase.resolves(apps); + }; + // Two g: components and one that runs on every instance; the anchor is db. + const twoElected = { + name: 'App', + version: 7, + compose: [ + { name: 'cache', containerData: '/cache' }, + { name: 'db', containerData: 'g:/data' }, + { name: 'web', containerData: 'g:/srv' }, + ], + }; + const oneElected = { name: 'Solo', version: 7, compose: [{ name: 'game', containerData: 'g:/data' }, { name: 'web', containerData: '/www' }] }; + describe('heldComponents', () => { // What a peer mid-election is told this node owns. Answering short here is not // a stale reading - it is a second container started on a volume this node is // already writing, which corrupts it. + beforeEach(() => { + installedAppsAre([]); + }); + // A primary back from a stop: its folder still sends, paused, and nothing runs // or is committed while it decides whether another holder took over. it('reports a component whose folder sends here, with nothing running, committed or stopped', async () => { @@ -645,8 +666,9 @@ describe('appQueryService tests', () => { sinon.stub(appReconciler, 'committedIdentifiers').returns(committed); sinon.stub(primaryRoleChanges, 'promotingIdentifiers').returns(promoting); sinon.stub(appsRuntimeState, 'operatorHeldIdentifiers').resolves(stopped); - messageHelperStub.createDataMessage.returnsArg(0); - return appQueryService.heldComponents(); + messageHelperStub.createDataMessage.callsFake((data) => ({ status: 'success', data })); + const answer = await appQueryService.heldComponents(); + return answer?.status === 'success' ? answer.data : answer; }; // A node becoming the primary has committed before its folder sends and long @@ -725,6 +747,62 @@ describe('appQueryService tests', () => { expect(result).to.deep.equal({ status: 'error' }); expect(messageHelperStub.createDataMessage.called, 'answered with a list built from a failed read').to.be.false; }); + + // A release that elects one g: component asks a peer only about the app's + // anchor. To it a node holding another of the app's g: components holds + // nothing, and it starts the app beside that node. + describe('an app with more than one g: component', () => { + beforeEach(() => { + installedAppsAre([twoElected, oneElected]); + }); + + it('reports the anchor of an app it runs a later g: component of', async () => { + const result = await held({ running: ['fluxweb_App'] }); + + expect(result).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + + it('reports the anchor of an app whose later g: component\'s folder alone sends here', async () => { + syncthingServiceStub.getConfigFolders.resolves([{ id: 'fluxweb_App', type: 'sendreceive' }]); + + const result = await held(); + + expect(result).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + + it('reports the anchor of an app it is becoming the primary of', async () => { + const result = await held({ promoting: ['web_App'] }); + + expect(result).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + + // The non-elected component runs on every instance: holding it is not + // holding the app. + it('adds nothing for a component that is not elected', async () => { + const result = await held({ running: ['fluxcache_App'] }); + + expect(result).to.deep.equal(['fluxcache_App']); + }); + + it('adds nothing for an app with one g: component', async () => { + const result = await held({ running: ['fluxweb_Solo'] }); + + expect(result).to.deep.equal(['fluxweb_Solo']); + }); + + it('adds nothing when it holds none of the app', async () => { + expect(await held()).to.deep.equal([]); + }); + }); + + it('fails the request when the installed apps cannot be read, rather than answering without the anchors', async () => { + dbHelperStub.findInDatabase.rejects(new Error('no primary available')); + messageHelperStub.createErrorMessage.returns({ status: 'error' }); + + const result = await held({ running: ['fluxweb_App'] }); + + expect(result).to.deep.equal({ status: 'error' }); + }); }); // The same account as a peer asks it: sealed - signed over the caller's @@ -736,6 +814,7 @@ describe('appQueryService tests', () => { const sealWith = (seal) => peerIdentityServiceStub.answerSealer.returns(seal); beforeEach(() => { + installedAppsAre([]); nowNs += 10n ** 10n; sinon.stub(process.hrtime, 'bigint').callsFake(() => nowNs); dockerServiceStub.dockerListContainers.resolves([{ Names: ['/fluxwww_App'] }]); @@ -755,6 +834,28 @@ describe('appQueryService tests', () => { sinon.assert.calledOnceWithExactly(peerIdentityServiceStub.answerSealer, 'held', req.body); }); + // A peer asking whether this node has DECIDED it holds the app reads the + // anchor there too, and only where a g: component of the app is decided. + it('carries the anchor in each list only where that list holds the app', async () => { + installedAppsAre([twoElected]); + dockerServiceStub.dockerListContainers.resolves([]); + syncthingServiceStub.getConfigFolders.resolves([{ id: 'fluxweb_App', type: 'sendreceive' }]); + + const answer = await appQueryService.heldComponentsAnswer({ body: {} }, res()); + + expect(answer.data.held).to.have.members(['fluxweb_App', 'fluxdb_App']); + expect(answer.data.decided).to.deep.equal([]); + }); + + it('carries the anchor as decided when a later g: component is decided here', async () => { + installedAppsAre([twoElected]); + dockerServiceStub.dockerListContainers.resolves([{ Names: ['/fluxweb_App'] }]); + + const answer = await appQueryService.heldComponentsAnswer({ body: {} }, res()); + + expect(answer.data.decided).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + it('holds a component whose folder alone sends here without having decided it', async () => { syncthingServiceStub.getConfigFolders.resolves([{ id: 'fluxgame_App', type: 'sendreceive' }]); diff --git a/tests/unit/mountParser.test.js b/tests/unit/mountParser.test.js index 05ac2206da..311031ef4b 100644 --- a/tests/unit/mountParser.test.js +++ b/tests/unit/mountParser.test.js @@ -440,32 +440,59 @@ describe('mountParser tests', () => { expect(result.additional[0].type).to.equal('component_primary'); }); }); - // The election decides one component per app, and an operator start defers to - // it only for that one: both read it here, so they cannot disagree. - describe('electedComponentIdentifier', () => { - it('names the first g: component of a composed app', () => { + // The election decides the app and runs every g: component on its primary; an + // operator start defers to it for the same set. Both read it here, so they + // cannot disagree. + describe('gComponentIdentifiers', () => { + it('names every g: component of a composed app, in compose order', () => { const spec = { name: 'App', version: 4, - compose: [{ name: 'db', containerData: '/data' }, { name: 'one', containerData: 'g:/data' }, { name: 'two', containerData: 'g:/data' }], + compose: [ + { name: 'two', containerData: 'g:/data' }, + { name: 'db', containerData: '/data' }, + { name: 'one', containerData: 'g:/data' }, + { name: 'cache', containerData: 'r:/data' }, + ], }; - expect(mountParser.electedComponentIdentifier(spec)).to.equal('one_App'); + expect(mountParser.gComponentIdentifiers(spec)).to.deep.equal(['two_App', 'one_App']); }); it('names a v1-3 g: app by its own name', () => { - expect(mountParser.electedComponentIdentifier({ name: 'Old', version: 3, containerData: 'g:/data' })).to.equal('Old'); + expect(mountParser.gComponentIdentifiers({ name: 'Old', version: 3, containerData: 'g:/data' })).to.deep.equal(['Old']); }); it('names nothing for an app with no g: component', () => { - expect(mountParser.electedComponentIdentifier({ name: 'Old', version: 3, containerData: 'r:/data' })).to.equal(null); - expect(mountParser.electedComponentIdentifier({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/www' }] })).to.equal(null); - expect(mountParser.electedComponentIdentifier({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/data|g:/x' }] })).to.equal(null); + expect(mountParser.gComponentIdentifiers({ name: 'Old', version: 3, containerData: 'r:/data' })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/www' }] })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/data|g:/x' }] })).to.deep.equal([]); }); it('names nothing for a spec whose components cannot be read', () => { - expect(mountParser.electedComponentIdentifier({ name: 'Ent', version: 8, enterprise: 'blob', compose: [] })).to.equal(null); - expect(mountParser.electedComponentIdentifier({ name: 'Ent', version: 8 })).to.equal(null); - expect(mountParser.electedComponentIdentifier(null)).to.equal(null); + expect(mountParser.gComponentIdentifiers({ name: 'Ent', version: 8, enterprise: 'blob', compose: [] })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers({ name: 'Ent', version: 8 })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers(null)).to.deep.equal([]); + }); + }); + + // The one g: component a release that elects a single component knows about. + describe('anchorComponentIdentifier', () => { + it('names the first g: component in compose order', () => { + const spec = { + name: 'App', + version: 4, + compose: [{ name: 'db', containerData: '/data' }, { name: 'one', containerData: 'g:/data' }, { name: 'two', containerData: 'g:/data' }], + }; + expect(mountParser.anchorComponentIdentifier(spec)).to.equal('one_App'); + }); + + it('names a v1-3 g: app by its own name', () => { + expect(mountParser.anchorComponentIdentifier({ name: 'Old', version: 3, containerData: 'g:/data' })).to.equal('Old'); + }); + + it('names nothing for an app with no g: component', () => { + expect(mountParser.anchorComponentIdentifier({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/www' }] })).to.equal(null); + expect(mountParser.anchorComponentIdentifier(null)).to.equal(null); }); }); }); diff --git a/tests/unit/peerComponent.test.js b/tests/unit/peerComponent.test.js index 0d760df30c..c2abcdd6bd 100644 --- a/tests/unit/peerComponent.test.js +++ b/tests/unit/peerComponent.test.js @@ -5,12 +5,17 @@ const { expect } = require('chai'); const sinon = require('sinon'); const axios = require('axios'); const peerIdentityService = require('../../ZelBack/src/services/peerIdentityService'); -const { PeerComponent, componentStateOnPeers, componentStateOnOtherHolders } = require('../../ZelBack/src/services/appMonitoring/peerComponent'); +const syncthingService = require('../../ZelBack/src/services/syncthingService'); +const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); +const globalState = require('../../ZelBack/src/services/utils/globalState'); +const { + PeerComponent, componentStateOnPeers, componentStateOnOtherHolders, peerComponentState, peerUndecided, +} = require('../../ZelBack/src/services/appMonitoring/peerComponent'); describe('peerComponent tests', () => { const appId = 'fluxw_a'; const ctx = { - appId, identifier: 'w_a', appName: 'a', liveness: {}, logPrefix: 'test', + appIds: [appId], identifier: 'w_a', appName: 'a', liveness: {}, logPrefix: 'test', }; // Each peer answers /apps/heldcomponents with what it holds; a peer given a @@ -105,4 +110,94 @@ describe('peerComponent tests', () => { expect(state).to.equal(PeerComponent.UNKNOWN); }); }); + + // An app with more than one g: component is held by the node that holds any of + // them: its primary runs them together. + describe('an app with more than one g: component', () => { + const appIds = ['fluxdb_a', 'fluxweb_a']; + const appCtx = { + appIds, identifier: 'db_a', appName: 'a', logPrefix: 'test', label: 'peer', + liveness: { localConnectivity: () => ({ connected: true }) }, + }; + const PEER = '2.2.2.2:16127'; + + it('reads a peer holding only a later g: component as holding the app', async () => { + peersAnswer({ [PEER]: { held: ['fluxweb_a'] } }); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.RUNNING); + }); + + it('reads a peer holding none of them as not holding the app', async () => { + peersAnswer({ [PEER]: { held: ['fluxweb_b', 'fluxother_a'] } }); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); + }); + + // A whole-name match, as for one component: another app whose component + // names merely contain these must not read as this app. + it('does not read a longer container name as one of the app\'s', async () => { + peersAnswer({ [PEER]: { held: ['fluxweb_a2', 'fluxdb_ab'] } }); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); + }); + + describe('a peer that does not answer', () => { + // This node's syncthing saw the peer after it started, and holds no live + // connection to it for whichever folders the test names. + const evidence = (connectedFolders) => { + sinon.stub(axios, 'get').rejects(new Error('connect ECONNREFUSED')); + sinon.stub(syncthingService, 'getConfigDevices').resolves([{ name: PEER, deviceID: 'PEERDEVICE' }]); + sinon.stub(syncthingService, 'getDbCompletion').callsFake(async ({ folder }) => ({ + remoteState: connectedFolders.includes(folder) ? 'valid' : 'notSharing', + })); + sinon.stub(syncthingService, 'getDeviceStats').resolves({ PEERDEVICE: { lastSeen: '2026-10-16T10:00:00Z' } }); + sinon.stub(syncthingService, 'getSystemStatus').resolves({ startTime: '2026-10-16T09:00:00Z' }); + sinon.stub(registryManager, 'shuttingDownNodes').resolves([]); + }; + + beforeEach(() => { + globalState.syncthingDevicesIDCache.clear(); + }); + + it('frees the app when its connection is gone for every g: folder', async () => { + evidence([]); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); + }); + + // Canary for the test above: the same peer with one folder still connected + // is one that may still be writing it. + it('does not free the app while one g: folder is still connected to it', async () => { + evidence(['fluxweb_a']); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); + }); + + it('does not free the app while the anchor\'s folder is still connected to it', async () => { + evidence(['fluxdb_a']); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); + }); + }); + + describe('peerUndecided', () => { + const answers = (decided) => { + peerIdentityService.askSigned.resolves({ + verdict: peerIdentityService.IdentityVerdict.VERIFIED, answer: { held: decided, decided }, + }); + }; + + it('reads a peer that has decided any of them as decided', async () => { + answers(['fluxweb_a']); + + expect(await peerUndecided(PEER, appCtx)).to.equal(false); + }); + + it('reads a peer that has decided none of them as undecided', async () => { + answers(['fluxweb_b']); + + expect(await peerUndecided(PEER, appCtx)).to.equal(true); + }); + }); + }); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 533abac319..e5e709f1a0 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -1047,9 +1047,23 @@ describe('syncthingFolderStateMachine tests', () => { const [readLocations, localSocketAddr, ctx, options] = peerComponentMock.componentStateOnOtherHolders.firstCall.args; expect(readLocations).to.equal(mockParams.readAppLocation); expect(localSocketAddr).to.equal(mockParams.localSocketAddr); - expect(ctx).to.include({ appId: 'test-app', identifier: 'app_test', appName: 'test-app' }); + expect(ctx).to.include({ identifier: 'app_test', appName: 'test-app' }); + expect(ctx.appIds).to.deep.equal(['test-app']); expect(options, 'a node FDM names').to.equal(undefined); }); + + // A holder running any g: component of the app is its primary, and this + // folder's unsent changes are not to go out beside it. + it('asks about every g: component of the app', async () => { + mockParams.readAppLocation = sinon.stub().resolves([]); + mockParams.electedSiblings = ['fluxweb_test-app']; + + const { othersHold } = await demote(); + await othersHold(); + + const [, , ctx] = peerComponentMock.componentStateOnOtherHolders.firstCall.args; + expect(ctx.appIds).to.deep.equal(['test-app', 'fluxweb_test-app']); + }); }); it('leaves the pause of a sending single-writer folder out of the write when what it holds cannot be read', async () => { @@ -2399,6 +2413,116 @@ describe('syncthingFolderStateMachine tests', () => { // node's address sorts LAST ('9.0.0.1' after '10.0.0.2' as a string), so under the // address order it loses and the empty peer publishes an empty folder over the // customer's world. It holds the data, so it must win. + // An app with more than one g: component is seeded on one node: its primary + // runs them all, so a folder seeded anywhere else leaves no node holding every + // copy it may run on. + describe('a g: folder of an app with more than one g: component', () => { + const SIBLING = 'fluxdb_test-app'; + // A brand-new folder: nothing indexed, nothing synced, nothing held here. + const freshFolder = ({ self = '10.0.0.1:16127', peer = '10.0.0.2:16127', peerAnswer } = {}) => { + mockParams.containerDataFlags = 'g'; + mockParams.electedSiblings = [SIBLING]; + mockParams.localSocketAddr = self; + mockParams.appLocation.resolves([ + { ip: self, runningSince: 2000, broadcastedAt: 1000 }, + { ip: peer, runningSince: 2000, broadcastedAt: 1000 }, + ]); + axiosMock.get.resolves({ data: { data: { ready: true, folders: [], holding: {}, ...peerAnswer } } }); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 0, inSyncBytes: 0, state: 'idle', receiveOnlyChangedFiles: 0, + }); + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { + restarted: false, numberOfExecutions: 1, leaderStreak: 5, + }); + }; + const seeds = (result) => result.cache.designationPending === true; + + afterEach(() => { + globalStateMock.folderHoldings?.delete(SIBLING); + }); + + describe('a component the spec has just added', () => { + // This node's address sorts last, so the address order seeds the peer. + const lastInOrder = { self: '9.0.0.1:16127', peer: '10.0.0.2:16127' }; + + it('is seeded by the node that holds the app\'s other g: component, wherever its address falls', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + // Canary for the test above: the same node holding none of the app leaves + // the seed to the address order. + it('is not seeded by a node holding none of the app when its address sorts last', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + it('is not seeded by the writer over a peer holding bytes of this folder', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 900, newestModified: 5 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + it('is left to a peer whose folder for the app\'s other g: component sends, though this node sorts first', async () => { + freshFolder({ peerAnswer: { folders: [SIBLING] } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + // Canary for the test above: with no writer anywhere, the node sorting + // first seeds. + it('is seeded by the node sorting first when no node writes the app', async () => { + freshFolder({ peerAnswer: { folders: [] } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + }); + + describe('a cold start where the holders hold different folders', () => { + // This node holds a little of this folder; the peer holds nothing of it + // and the whole of the other one. + const split = () => { + freshFolder({ + peerAnswer: { + holding: { 'test-app': { bytes: 0, newestModified: 0 }, [SIBLING]: { bytes: 5000, newestModified: 9 } }, + }, + }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [localEntry('a.dat', 100)] })); + globalStateMock.folderHoldings = globalStateMock.folderHoldings || new Map(); + globalStateMock.folderHoldings.set(SIBLING, { bytes: 0, newestModified: 0 }); + }; + + it('ranks the whole app, so the node holding the most of it seeds every folder', async () => { + split(); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + // Canary for the test above: ranked on this folder alone, this node wins. + it('would seed this folder here ranked on this folder alone', async () => { + split(); + mockParams.electedSiblings = []; + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + it('falls back to the address order while this node cannot say what it holds of the other folder', async () => { + split(); + globalStateMock.folderHoldings.delete(SIBLING); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + }); + }); + it('the node holding the data wins the election though its address sorts last', async () => { mockParams.localSocketAddr = '9.0.0.1:16127'; syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [localEntry('world.sav', 5821604997, '2026-09-16T09:00:00Z')] })); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index d985521bf8..e0c30efc18 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -1591,6 +1591,32 @@ describe('syncthingMonitor tests', () => { expect(failure?.message).to.equal('database unavailable'); }); + // The state machine seeds an app's g: folders on one node, so each is told + // the folders of the app's other g: components; a folder of any other + // component is told none. + it('hands each g: folder the folders of the app\'s other g: components', async () => { + writesAFolder(); + mockInstalledAppsFn.resolves({ + status: 'success', + data: [{ + name: 'twin', + version: 7, + compose: [ + { name: 'db', containerData: 'g:/data' }, + { name: 'cache', containerData: 'r:/cache' }, + { name: 'web', containerData: 'g:/srv' }, + ], + }], + }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + const siblings = Object.fromEntries(syncthingFolderStateMachineMock.manageFolderSyncState.getCalls() + .map((call) => [call.args[0].appId, call.args[0].electedSiblings])); + expect(siblings).to.deep.equal({ db_twin: ['web_twin'], cache_twin: [], web_twin: ['db_twin'] }); + }); + describe('the type of a g: folder', () => { const folderCalls = () => syncthingServiceMock.adjustConfigFolders.getCalls() .filter((c) => c.args[0] === 'put' || c.args[0] === 'patch'); From 41187e3ab02c1eb7722592ebfb997c29d7a9bd12 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:30:09 +0100 Subject: [PATCH 015/369] test(harness): 2242 is an update that adds a g: component, which reinstalls the app A change to an app's components reinstalls it on every holder, so the suite asserts what that leaves: the app never runs on two nodes, and settles with both g: components on one. The added component takes its port from the allocator, as registration gives one. Co-Authored-By: Claude Opus 5.5 --- ...a-g-component-runs-the-app-on-one-node.js} | 70 ++++++++++--------- 1 file changed, 37 insertions(+), 33 deletions(-) rename test-infra/runner/tests/{2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js => 2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js} (58%) diff --git a/test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js similarity index 58% rename from test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js rename to test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js index 9e803a6a97..952ddd3a51 100644 --- a/test-infra/runner/tests/2242-a-g-component-a-spec-update-adds-is-started-by-the-primary.js +++ b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js @@ -5,6 +5,7 @@ import { createTestEnv } from '../framework/test-env.js'; import { listAppContainers } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { assignPorts } from '../framework/port-allocator.js'; import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; import { waitFor, waitHolding } from '../framework/wait.js'; import { driveUntil, stopTicker } from '../framework/daemon-control.js'; @@ -12,32 +13,29 @@ import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncth import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; -// A specification update adds a second single-writer (g:) component to a running -// app, on real syncthing daemons. +// A specification update gives a running app a second single-writer (g:) +// component, on real syncthing daemons. // -// Its folder is new on every holder and holds nothing anywhere, and the app's -// primary is the holder whose address sorts LAST. Seeded by the address order the -// new folder would be the standby's, and the primary - which runs every g: -// component of the app - would never hold a copy it may start the new one on. The -// primary seeds it and starts it, the standby never runs it, and at no point does -// any g: component of the app run on two nodes. +// A change to the components is a structure change: each holder removes the app +// and installs it at the new specification, on its own reinstall pass. Between the +// two passes one holder is at the new specification and the other still runs the +// old one, and after both the app starts afresh. Through all of it no two nodes run +// any g: component of the app at once, and once settled one node runs both and the +// other runs neither. // // DRIVEN BY BLOCKS: the reinstall pass fires on // `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). const APP_UID = 1000; -describe('a g: component a spec update adds is started by the primary', function () { +describe('an update that adds a g: component runs the app on one node', function () { let env; dumpLogsOnFailure(() => env); const appName = `e2eaddg${Date.now()}`; - const folderOf = (component) => `flux${component}_${appName}`; - // The standby sorts first by address; the primary is placed first, so it runs - // the app, and sorts last. - const STANDBY = 0; - const PRIMARY = 1; - const HOLDERS = [STANDBY, PRIMARY]; + const COMPONENTS = ['db', 'web']; + const folderOf = (name) => `flux${name}_${appName}`; + const HOLDERS = [0, 1]; let app; const client = (i) => env.clients[i]; @@ -58,7 +56,7 @@ describe('a g: component a spec update adds is started by the primary', function }); const runningOn = async (i) => { const containers = await listAppContainers(client(i).container); - return ['db', 'web'].filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); + return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); }; // Which holders run any of the app's g: components; asserted on every poll. const oneHolderAtMost = async () => { @@ -83,19 +81,18 @@ describe('a g: component a spec update adds is started by the primary', function await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, }))); - expect(`${client(STANDBY).ip}:16127` < `${client(PRIMARY).ip}:16127`, `fixture: the standby (${client(STANDBY).ip}) sorts before the primary (${client(PRIMARY).ip}) by address`).to.equal(true); await pushTestApp(appName, 'v1', 'addg', { user: APP_UID }); app = await buildSeedableApp({ env, name: appName, instances: HOLDERS.length, compose: [component('db')], }); - await installOnNodes(env, app, [PRIMARY]); - await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { - timeout: 300000, interval: 3000, label: 'the primary runs db', + await installOnNodes(env, app, [HOLDERS[0]]); + await waitHolding(async () => (await oneHolderAtMost())[HOLDERS[0]].includes('db'), { + timeout: 300000, interval: 3000, label: 'the first holder runs db', }); - await installOnNodes(env, app, [STANDBY]); - await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { - timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + await installOnNodes(env, app, [HOLDERS[1]]); + await waitFor(() => isFolderSynced(client(HOLDERS[1]), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the second holder has the primary\'s db data', }); }); @@ -104,33 +101,40 @@ describe('a g: component a spec update adds is started by the primary', function await env?.teardown(); }); - it('seeds the new component\'s folder on the primary and starts it there, never on the standby', async function () { + it('never runs the app on two nodes, and settles with both g: components on one', async function () { // A backstop: the block drive bounds this test (suite 85). this.timeout(1800000); const updated = await buildSeedableUpdate(app, (spec) => { spec.compose.push(component('web')); + // The new component's port, as registration allocates one; db keeps its own. + assignPorts(spec.compose, appName); }); await seedSpecUpdate(env, updated, HOLDERS); await stopTicker(); let breach = null; - await driveUntil(client(PRIMARY), async () => { + let settled = null; + await driveUntil(client(HOLDERS[0]), async () => { try { const running = await oneHolderAtMost(); - return running[PRIMARY].length === 2; + settled = HOLDERS.find((i) => running[i].length === COMPONENTS.length) ?? null; + return settled !== null; } catch (error) { if (error?.name !== 'AssertionError') throw error; breach = error; return true; } - }, { blocks: 216, label: 'the primary runs db and web' }); + }, { blocks: 216, label: 'one node runs db and web' }); if (breach) throw breach; - expect((await getFolderConfig(client(PRIMARY), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); - await waitFor(() => isFolderSynced(client(STANDBY), folderOf('web')), { - timeout: 300000, interval: 3000, label: 'the standby has the primary\'s web data', - }); - expect((await getFolderConfig(client(STANDBY), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); - expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + const other = HOLDERS.find((i) => i !== settled); + expect((await getFolderConfig(client(settled), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + expect(running[settled], 'the primary').to.deep.equal(COMPONENTS); + return isFolderSynced(client(other), folderOf('web')); + }, { timeout: 300000, interval: 3000, label: 'the standby has the primary\'s web data' }); + expect((await getFolderConfig(client(other), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + expect((await oneHolderAtMost())[other], 'the standby').to.deep.equal([]); }); }); From 7b6e34c5d0df0b25a7ddb664e87f8cac2a01c2af Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:33:45 +0100 Subject: [PATCH 016/369] test(harness): 2242 allocates the added component's port the way registration allocates the first's Co-Authored-By: Claude Opus 5.5 --- ...ate-that-adds-a-g-component-runs-the-app-on-one-node.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js index 952ddd3a51..c5ea5f94ab 100644 --- a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js +++ b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js @@ -5,7 +5,7 @@ import { createTestEnv } from '../framework/test-env.js'; import { listAppContainers } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; -import { assignPorts } from '../framework/port-allocator.js'; +import { allocatePortFor } from '../framework/port-allocator.js'; import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; import { waitFor, waitHolding } from '../framework/wait.js'; import { driveUntil, stopTicker } from '../framework/daemon-control.js'; @@ -105,9 +105,8 @@ describe('an update that adds a g: component runs the app on one node', function // A backstop: the block drive bounds this test (suite 85). this.timeout(1800000); const updated = await buildSeedableUpdate(app, (spec) => { - spec.compose.push(component('web')); - // The new component's port, as registration allocates one; db keeps its own. - assignPorts(spec.compose, appName); + // The new component's port, allocated as registration allocates db's. + spec.compose.push({ ...component('web'), ports: [allocatePortFor(appName, 1)] }); }); await seedSpecUpdate(env, updated, HOLDERS); await stopTicker(); From bcc8d9423044c70c7aedc6b267c3b323b396711d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:42:13 +0100 Subject: [PATCH 017/369] test(harness): 2242 drives the chain until both holders have reinstalled Co-Authored-By: Claude Opus 5.5 --- ...hat-adds-a-g-component-runs-the-app-on-one-node.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js index c5ea5f94ab..367662ce72 100644 --- a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js +++ b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js @@ -54,6 +54,11 @@ describe('an update that adds a g: component runs the app on one node', function hdd: 1, repoauth: '', }); + // The specification a holder has installed, by its hash. + const installedHash = async (i) => { + const res = await client(i).getInstalledApps(); + return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; + }; const runningOn = async (i) => { const containers = await listAppContainers(client(i).container); return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); @@ -113,17 +118,19 @@ describe('an update that adds a g: component runs the app on one node', function let breach = null; let settled = null; + // Until both holders have reinstalled: each does on its own pass. await driveUntil(client(HOLDERS[0]), async () => { try { const running = await oneHolderAtMost(); settled = HOLDERS.find((i) => running[i].length === COMPONENTS.length) ?? null; - return settled !== null; + const hashes = await Promise.all(HOLDERS.map(installedHash)); + return settled !== null && hashes.every((hash) => hash === updated.hash); } catch (error) { if (error?.name !== 'AssertionError') throw error; breach = error; return true; } - }, { blocks: 216, label: 'one node runs db and web' }); + }, { blocks: 216, label: 'both holders are at the new specification and one runs db and web' }); if (breach) throw breach; const other = HOLDERS.find((i) => i !== settled); From 011a17bd9aa6afbe7bf17cc1dbd454513926b2bb Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 14:03:07 +0100 Subject: [PATCH 018/369] test(harness): a node can boot another commit's image and be upgraded in place createTestEnv takes a per-node image tag, so a fleet can mix commits; an image under another tag must exist on the host and carry build-images.sh's source stamp. env.upgradeNode copies the FluxOS code of another built image into a running node, its installed packages when the build declares others, and writes its config overlay from the new build's per-node file and the run's overrides with write-node-config.sh, the step the entrypoint runs at boot; its data, containers and database stay, and FluxOS restarts as on a node's own update. build-release-image.sh builds a node image whose FluxOS is another ref's product code inside this tree's harness. A suite that boots a node on such an image declares `// base-image: FLUX_E2E_BASE_TAG`, and run-parallel refuses (91) before launching when that image is not on the host. Ported from the v9 lineage (6646baa45d, 217e1c0a66, 1d6bb3c55d) with v9's text, less what this tree does not have: the vendored /flux-spec and ZelBack/configReconciliation.js are not copied by upgradeNode, and the node image name is a local constant where v9 imports it from flux-spec-vendor.js. Co-Authored-By: Claude Opus 5.5 --- test-infra/build-release-image.sh | 48 +++++++++ test-infra/entrypoint.sh | 33 +----- test-infra/runner/framework/test-env.js | 131 ++++++++++++++++++++++-- test-infra/runner/run-parallel.sh | 16 +++ test-infra/write-node-config.sh | 36 +++++++ 5 files changed, 223 insertions(+), 41 deletions(-) create mode 100755 test-infra/build-release-image.sh create mode 100755 test-infra/write-node-config.sh diff --git a/test-infra/build-release-image.sh b/test-infra/build-release-image.sh new file mode 100755 index 0000000000..1f36f3909f --- /dev/null +++ b/test-infra/build-release-image.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Builds a node image whose FluxOS is another ref's product code inside this tree's +# harness - the stubs, entrypoint and runner a fleet of this tree boots with - so a +# suite can boot a node on a released FluxOS and upgrade it in place to this tree's. +# +# FLUX_E2E_TAG= ./test-infra/build-release-image.sh +# +# The ref must be in this clone. Builds fluxos-01 only, stamped like any other +# (build-images.sh), from a temporary worktree of HEAD with the product paths +# taken from the ref and the per-node configs generated from the ref's own config. +# The gitignored build products the image needs are copied from this clone. +set -euo pipefail + +REF=${1:?usage: FLUX_E2E_TAG= $0 } +: "${FLUX_E2E_TAG:?FLUX_E2E_TAG names the tag to build under}" +CLONE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PRODUCT=(ZelBack helpers lib app.js apiServer.js init.js package.json) + +git -C "$CLONE" rev-parse --verify --quiet "$REF^{commit}" >/dev/null || { echo "build-release-image: $REF is not in this clone" >&2; exit 2; } + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/flux-release-image.XXXXXX") +cleanup() { + git -C "$CLONE" worktree remove --force "$WORK/tree" >/dev/null 2>&1 || true + rm -rf "$WORK" +} +trap cleanup EXIT + +git -C "$CLONE" worktree add --quiet --detach "$WORK/tree" HEAD +cd "$WORK/tree" + +for p in "${PRODUCT[@]}"; do + rm -rf "$p" + git archive "$REF" -- "$p" | tar -x +done + +# Whatever the harness built and ignores: vendored libraries, fixture binaries. +git -C "$CLONE" ls-files --others --ignored --exclude-standard -- test-infra \ + | grep -v -e '/node_modules/' -e '^test-infra/runner/' \ + | while read -r f; do mkdir -p "$(dirname "$f")"; cp -p "$CLONE/$f" "$f"; done + +# The per-node configs a node of the ref's code reads, from the ref's production config. +mkdir -p config +cp test-infra/fixtures/userconfig-template.js config/userconfig.js +node test-infra/config/generate-configs.js +rm -f config/userconfig.js + +./test-infra/build-images.sh fluxos-01 +echo "build-release-image: flux-e2e-fluxos-01:$FLUX_E2E_TAG holds $REF ($(git -C "$CLONE" rev-parse --short "$REF^{commit}"))" diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 71e3d7b9f6..e330cfbf27 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -59,38 +59,9 @@ if [ "$FLUX_SYNCTHING_MODE" != "binary" ]; then cp /flux/test-infra/fixtures/syncthing-config.xml /dat/usr/lib/syncthing/config.xml 2>/dev/null || true fi -# Overlay test config into ZelBack/config/ so app.js loads it naturally. -# app.js hardcodes NODE_CONFIG_DIR to ZelBack/config/ (cannot be overridden -# from env — fluxbenchd hashes that directory for tamper detection). -if [ -n "$NODE_CONFIG_DIR" ] && [ -d "$NODE_CONFIG_DIR" ]; then - cp "$NODE_CONFIG_DIR"/default.js /flux/ZelBack/config/local.js - cp "$(dirname "$NODE_CONFIG_DIR")/shared.js" /flux/ZelBack/ 2>/dev/null || true -fi +# The node's config overlay: its per-node file with the runner's overrides over it. +/flux/test-infra/write-node-config.sh -# The runner's own overrides arrive as JSON and are merged OVER the per-node file -# copied above, which is where the per-node database names come from - replacing -# that file rather than merging would take them with it. -# -# They used to arrive as NODE_CONFIG. The config package merges that variable over -# every file whatever directory is pinned, so it could redirect any endpoint -# without touching the directory fluxbenchd hashes - the one change tamper -# detection cannot see. The entry points delete it now, and this carries the same -# content to the same place through a file instead. -if [ -n "$FLUX_TEST_CONFIG" ]; then - node -e ' - const fs = require("fs"); - const target = "/flux/ZelBack/config/local.js"; - const base = fs.existsSync(target) ? require(target) : {}; - const isPlain = (v) => v && typeof v === "object" && !Array.isArray(v); - const merge = (a, b) => { - const out = { ...a }; - for (const [k, v] of Object.entries(b)) out[k] = isPlain(v) && isPlain(a[k]) ? merge(a[k], v) : v; - return out; - }; - const merged = merge(base, JSON.parse(process.env.FLUX_TEST_CONFIG)); - fs.writeFileSync(target, `module.exports = ${JSON.stringify(merged, null, 2)};\n`); - ' -fi # A node whose discovery the harness has started rejoins the mesh by itself on # every later start, the run's own discoveryAutostart notwithstanding. node /flux/test-infra/rejoin-discovery.cjs apply diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 2a269a7a09..f3b885adb8 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -6,7 +6,7 @@ process.env.TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT ??= '5s'; import { GenericContainer, Wait, getContainerRuntimeClient } from 'testcontainers'; import { - readFileSync, mkdirSync, writeFileSync, rmSync, existsSync, + readFileSync, mkdirSync, mkdtempSync, writeFileSync, rmSync, existsSync, } from 'node:fs'; import { execFile, execFileSync } from 'node:child_process'; import { tmpdir } from 'node:os'; @@ -14,7 +14,7 @@ import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import http from 'node:http'; import { nodeClient } from './node-client.js'; -import { execInContainer } from './container.js'; +import { execInContainer, restartFluxos } from './container.js'; import { waitFor } from './wait.js'; import { HttpPollWaitStrategy } from './http-wait-strategy.js'; import { TcpPollWaitStrategy } from './tcp-wait-strategy.js'; @@ -42,6 +42,14 @@ import { startSyncedDataKeeper } from './synced-data-keeper.js'; // replaying a line it already has. Docker's `since` is whole-second, so the replay is over // within a second of the stream opening; this is that bound with room for a loaded box. const REPLAY_WINDOW_MS = 5000; +// Bounded docker CLI call from the runner host. +function dockerCli(args, timeoutMs = 15000, maxBuffer = 8 * 1024 * 1024) { + return new Promise((resolve) => { + execFile('docker', args, { timeout: timeoutMs, maxBuffer }, (err, stdout, stderr) => { + resolve({ ok: !err, stdout: stdout ?? '', stderr: stderr ?? '' }); + }); + }); +} // A node's whole record, never a tail: the journal directory is the node's own, so // it holds exactly this node's entries since its container was created, and a @@ -208,6 +216,43 @@ const IMAGE_TAG = process.env.FLUX_E2E_TAG || 'latest'; const image = (name) => `${name}:${IMAGE_TAG}`; +const NODE_IMAGE = 'flux-e2e-fluxos-01'; + +// The node image a node boots: the run's tag, or the tag a suite names for that +// node, so a fleet can mix commits. An image under another tag is built from +// its own tree, which is not here to check it against, so what can be required +// is that it exists and was stamped by build-images.sh. +const nodeImageRef = (tag) => `${NODE_IMAGE}:${tag}`; + +async function assertNodeImageStamped(tag) { + const ref = nodeImageRef(tag); + const res = await dockerCli(['image', 'inspect', ref, '--format', '{{index .Config.Labels "flux.e2e.src"}}']); + if (!res.ok) { + throw new Error(`node image ${ref} is not built on this host: build it from its commit's clone with FLUX_E2E_TAG=${tag} ./test-infra/build-images.sh fluxos-01`); + } + const label = res.stdout.trim(); + if (!label || label === '') throw new Error(`node image ${ref} carries no source stamp: build it with build-images.sh, never by hand`); + return label; +} + +// The FluxOS code of one built node image, copied into a running node: the +// parts of the tree the process loads. Its data, its containers and its +// database stay. +const UPGRADE_DIRS = ['ZelBack/src', 'helpers', 'lib']; +const UPGRADE_FILES = ['package.json', 'ZelBack/pinEnvironment.js', 'ZelBack/config/default.js', 'ZelBack/config/volumeToolsImage.json', 'app.js', 'apiServer.js', 'init.js']; +// The new build's per-node config files and the script that writes the overlay from them. +const UPGRADE_HARNESS = ['test-infra/config', 'test-infra/write-node-config.sh']; + +async function dockerCp(from, to) { + const res = await dockerCli(['cp', from, to], 180000); + if (!res.ok) throw new Error(`docker cp ${from} ${to}: ${res.stderr.trim()}`); +} + +function dependenciesOf(packageJsonPath) { + const pkg = JSON.parse(readFileSync(packageJsonPath, 'utf8')); + return JSON.stringify({ dependencies: pkg.dependencies ?? {}, optionalDependencies: pkg.optionalDependencies ?? {} }); +} + // Where this run's log output lives. run-all.sh exports the resolved value; a // standalone `npx mocha` has no runner to inherit from and gets the same default // the runner would have applied, so both paths write to one place. @@ -925,7 +970,7 @@ export async function createTestEnv({ tickerAutostart = false, discoveryAutostart = false, nodeStatusOverrides = {}, rpcFailures = [], bootContext = 'running', initialHeight = DEFAULT_INITIAL_HEIGHT, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation = {}, locationTable = null, staticIp = true, policy = null, policySeeds = null, - awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], + awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], // systemd-mode nodes: the entrypoint execs a real systemd as PID 1, so dockerd // and fluxos run as units and FluxOS's output goes to the journal. Env-level // and uniform across nodes. @@ -1237,7 +1282,7 @@ export async function createTestEnv({ // mongo starts, i.e. inside the fleet boot, where the waits at risk are the // boot's own. await startInfraDeathWatch(env); - await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, systemdMode, firewall, collateralOf, syncthingFaults, configuredIp); + await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, systemdMode, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags); return env; } catch (err) { // Boot failed: the env owns everything started so far. The shared teardown @@ -1266,7 +1311,7 @@ function mergeConfigs(base, override) { return result; } -async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], systemdMode = false, firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}) { +async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], systemdMode = false, firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}) { // Everything built here registers onto the env shell as it comes up, so a // boot-phase throw leaves the partial state reachable (see makeEnvShell). const { @@ -1548,6 +1593,8 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod const deferredBuilders = new Map(); const firstDeferred = nodes - deferredNodes; + // A node booting another commit's image needs that image here, stamped. + await Promise.all([...new Set(Object.values(nodeImageTags))].filter((t) => t !== IMAGE_TAG).map(assertNodeImageStamped)); for (let i = 0; i < nodes; i++) { if (stubPeerSet.has(i)) continue; @@ -1751,7 +1798,8 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // state machine: under a contended 10-node fleet boot, Wait.forHealthCheck() // tears the fleet down on a transient "unhealthy" even when FluxOS is up. See // http-wait-strategy.js for the full rationale. - const builder = new StaticIpContainer(image('flux-e2e-fluxos-01')) + const imageTag = nodeImageTags[i] ?? IMAGE_TAG; + const builder = new StaticIpContainer(nodeImageRef(imageTag)) .withPrivilegedMode() .withStaticIp(networkName, nodeIp) // Nodes resolve through the stub, which answers fleet names by relaying to @@ -1769,7 +1817,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod if (nodeAppArmor()) builder.withAppArmorProfile(nodeAppArmor()); nodeConfigs.push({ - index: i, builder, ip: nodeIp, num: i + 1, logCollector, bootIdDir, journalDir, + index: i, builder, ip: nodeIp, num: i + 1, logCollector, bootIdDir, journalDir, imageTag, }); } @@ -1818,15 +1866,21 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod for (let i = 0; i < nodes; i++) { const cfg = fluxNodesByIndex.get(i); if (!cfg) { - fluxNodes.push({ container: null, ip: subnet.nodeIp(i + 1), num: i + 1, logCollector: null, bootIdDir: null }); + fluxNodes.push({ + container: null, ip: subnet.nodeIp(i + 1), num: i + 1, logCollector: null, bootIdDir: null, imageTag: IMAGE_TAG, + }); continue; } const s = startedByIndex.get(i); if (s) { - fluxNodes.push({ container: s.container, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir }); + fluxNodes.push({ + container: s.container, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir, imageTag: cfg.imageTag, + }); } else { deferredBuilders.set(i, cfg.builder); - fluxNodes.push({ container: null, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir }); + fluxNodes.push({ + container: null, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir, imageTag: cfg.imageTag, + }); } } containers.fluxNodes = fluxNodes; @@ -1967,6 +2021,63 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod return client; }, + // The image tag a node's FluxOS code came from: the one it booted, or the + // one it was last upgraded to. + nodeImageTag(index) { + return fluxNodes[index]?.imageTag ?? IMAGE_TAG; + }, + + // Make a running node run the FluxOS of another built image and restart + // FluxOS on it, as a node's own update does: its data, containers and database + // stay; its code, its installed packages when the build declares others, and + // its config overlay become the new build's - the overlay written from that + // build's per-node file and this run's overrides, by the script the entrypoint + // runs at boot. Every migration runs at the first start of the new code. + // Upgrade or downgrade alike. + async upgradeNode(index, { tag, readyTimeoutMs = 180000 } = {}) { + const node = fluxNodes[index]; + if (!node?.container) throw new Error(`upgradeNode: node ${index} is not running`); + if (!tag) throw new Error('upgradeNode: the image tag to run is required'); + await assertNodeImageStamped(tag); + const nodeId = node.container.getId(); + const created = await dockerCli(['create', nodeImageRef(tag)], 60000); + if (!created.ok) throw new Error(`upgradeNode: cannot read ${nodeImageRef(tag)}: ${created.stderr.trim()}`); + const sourceId = created.stdout.trim(); + const host = mkdtempSync(join(tmpdir(), 'flux-e2e-upgrade-')); + try { + await dockerCp(`${sourceId}:/flux/package.json`, `${host}/source-package.json`); + await dockerCp(`${nodeId}:/flux/package.json`, `${host}/node-package.json`); + const packagesChange = dependenciesOf(`${host}/source-package.json`) !== dependenciesOf(`${host}/node-package.json`); + const replaced = [...UPGRADE_DIRS, ...(packagesChange ? ['node_modules'] : [])]; + mkdirSync(`${host}/flux/ZelBack/config`, { recursive: true }); + mkdirSync(`${host}/flux/test-infra`, { recursive: true }); + for (const part of [...replaced, ...UPGRADE_FILES, ...UPGRADE_HARNESS]) { + // eslint-disable-next-line no-await-in-loop + await dockerCp(`${sourceId}:/flux/${part}`, `${host}/flux/${part}`); + } + const clear = [...replaced, ...UPGRADE_HARNESS].map((d) => `rm -rf /flux/${d}`).join(' && '); + const cleared = await execInContainer(node.container, clear); + if (cleared.exitCode !== 0) throw new Error(`upgradeNode: cannot clear node ${index}'s code: ${cleared.output}`); + for (const part of [...replaced, ...UPGRADE_HARNESS]) { + // eslint-disable-next-line no-await-in-loop + await dockerCp(`${host}/flux/${part}`, `${nodeId}:/flux/${dirname(part)}/`); + } + for (const file of UPGRADE_FILES) { + // eslint-disable-next-line no-await-in-loop + await dockerCp(`${host}/flux/${file}`, `${nodeId}:/flux/${file}`); + } + const owned = [...replaced, ...UPGRADE_FILES].map((p) => `/flux/${p}`).join(' '); + const configured = await execInContainer(node.container, `/flux/test-infra/write-node-config.sh && chown -R fluxuser:fluxuser ${owned} /flux/ZelBack/config && (chown fluxuser:fluxuser /flux/ZelBack/shared.js 2>/dev/null || true)`); + if (configured.exitCode !== 0) throw new Error(`upgradeNode: cannot write node ${index}'s config: ${configured.output}`); + } finally { + await dockerCli(['rm', '-f', sourceId], 60000); + rmSync(host, { recursive: true, force: true }); + } + await restartFluxos(node.container, { readyTimeoutMs }); + node.imageTag = tag; + }, + + // Wait on an HTTP poll of /flux/version rather than Docker's health state // machine: on restart Docker transiently reports "unhealthy" during monitor // teardown (moby/daemon/container/health.go CloseMonitorChannel), which a diff --git a/test-infra/runner/run-parallel.sh b/test-infra/runner/run-parallel.sh index 0e2c7f7fde..b0dc6b947f 100755 --- a/test-infra/runner/run-parallel.sh +++ b/test-infra/runner/run-parallel.sh @@ -175,6 +175,22 @@ if [ -z "${SUITES:-}" ]; then SUITES="$heavy $light" fi +# A suite that boots a node on another build of FluxOS declares it on a line of its +# own (`// base-image: FLUX_E2E_BASE_TAG`); the gate refuses before launching anything +# when a chosen suite declares one and that image is not on this host, rather than +# letting the suite fail inside the run. +for s in $SUITES; do + grep -qh '^// base-image: FLUX_E2E_BASE_TAG$' tests/${s}-*.js 2>/dev/null || continue + if [ -z "${FLUX_E2E_BASE_TAG:-}" ]; then + abort 91 "###ABORT suite $s boots a node on a base image and FLUX_E2E_BASE_TAG is not set." \ + "Build one: FLUX_E2E_TAG= ./test-infra/build-release-image.sh , then run with FLUX_E2E_BASE_TAG=." + fi + if ! docker image inspect "flux-e2e-fluxos-01:${FLUX_E2E_BASE_TAG}" >/dev/null 2>&1; then + abort 91 "###ABORT suite $s boots a node on flux-e2e-fluxos-01:${FLUX_E2E_BASE_TAG}, which is not on this host." \ + "Build it: FLUX_E2E_TAG=${FLUX_E2E_BASE_TAG} ./test-infra/build-release-image.sh ." + fi +done + free_mb(){ free -m | awk '/^Mem:/{print $7}'; } load_1m(){ cut -d' ' -f1 /proc/loadavg | cut -d. -f1; } # integer part is enough for a gate diff --git a/test-infra/write-node-config.sh b/test-infra/write-node-config.sh new file mode 100755 index 0000000000..52e0df8727 --- /dev/null +++ b/test-infra/write-node-config.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Writes the node's FluxOS config overlay, ZelBack/config/local.js: the per-node file +# NODE_CONFIG_DIR names, with the runner's overrides (FLUX_TEST_CONFIG, JSON) merged +# over it. Run by the entrypoint at boot and by an in-place upgrade, which takes the +# new build's per-node file the same way. +# +# app.js hardcodes NODE_CONFIG_DIR to ZelBack/config/ (cannot be overridden from env +# - fluxbenchd hashes that directory for tamper detection), so the test config is +# overlaid into it. +set -e + +if [ -n "$NODE_CONFIG_DIR" ] && [ -d "$NODE_CONFIG_DIR" ]; then + cp "$NODE_CONFIG_DIR"/default.js /flux/ZelBack/config/local.js + cp "$(dirname "$NODE_CONFIG_DIR")/shared.js" /flux/ZelBack/ 2>/dev/null || true +fi + +# Merged OVER the per-node file, which is where the per-node database names come +# from - replacing that file rather than merging would take them with it. A file +# rather than NODE_CONFIG: the config package merges that variable over every file +# whatever directory is pinned, so it could redirect any endpoint without touching +# the directory fluxbenchd hashes, and the entry points delete it. +if [ -n "$FLUX_TEST_CONFIG" ]; then + node -e ' + const fs = require("fs"); + const target = "/flux/ZelBack/config/local.js"; + const base = fs.existsSync(target) ? require(target) : {}; + const isPlain = (v) => v && typeof v === "object" && !Array.isArray(v); + const merge = (a, b) => { + const out = { ...a }; + for (const [k, v] of Object.entries(b)) out[k] = isPlain(v) && isPlain(a[k]) ? merge(a[k], v) : v; + return out; + }; + const merged = merge(base, JSON.parse(process.env.FLUX_TEST_CONFIG)); + fs.writeFileSync(target, `module.exports = ${JSON.stringify(merged, null, 2)};\n`); + ' +fi From b003ff6e0a8881dcad3a5b648fbd737c98013cfb Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 14:04:45 +0100 Subject: [PATCH 019/369] test(harness): 2243 and 2244 upgrade a two-g: app's holders from the release, in either order Two nodes on the release hold an app with two g: components, so the primary runs only the first; its address sorts after the standby's. 2243 upgrades the standby first, which starts nothing while the primary on the release holds the app, then the primary, which starts the second component. 2244 upgrades the primary first, which starts the second component while the standby is still on the release, then the standby, which changes nothing. Every poll holds that no two nodes run any g: component of the app. Co-Authored-By: Claude Opus 5.5 --- ...andby-first-brings-up-every-g-component.js | 151 ++++++++++++++++++ ...imary-first-brings-up-every-g-component.js | 151 ++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js create mode 100644 test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js diff --git a/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js b/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js new file mode 100644 index 0000000000..f94d397124 --- /dev/null +++ b/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js @@ -0,0 +1,151 @@ +// base-image: FLUX_E2E_BASE_TAG +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { listAppContainers } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, held by two nodes on a release +// that starts only the first, upgraded to this tree one node at a time: the +// standby first, then the primary. Real syncthing daemons. +// +// On the release the primary runs the first g: component and never the second. +// With the standby upgraded and the primary not, nothing changes: the upgraded +// standby stands aside for a primary that holds the app, and starts nothing. With +// the primary upgraded too it starts the second component beside the first. The +// primary's address sorts after the standby's, so the second component's folder is +// seeded by the node that writes the app, not by the address order. At no point +// does any g: component of the app run on two nodes. +// +// Node 2 runs this tree throughout and holds nothing: the third node of the ring. +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + +const APP_UID = 1000; +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; + +describe('a rolling upgrade, standby first, brings up every g: component', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eroll${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const folderOf = (name) => `flux${name}_${appName}`; + const STANDBY = 0; + const PRIMARY = 1; + const RING = 2; + const HOLDERS = [STANDBY, PRIMARY]; + + const client = (i) => env.clients[i]; + const runningOn = async (i) => { + const containers = await listAppContainers(client(i).container); + return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); + }; + // Which holders run any of the app's g: components; asserted on every poll. + const oneHolderAtMost = async () => { + const running = await Promise.all(HOLDERS.map(runningOn)); + const holders = HOLDERS.filter((_, k) => running[k].length); + expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); + return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); + }; + // Held over a minute of election passes, not read once. + const holdsFor = async (ms, expected) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + const running = await oneHolderAtMost(); + expect(running, 'what each holder runs').to.deep.equal(expected); + // eslint-disable-next-line no-await-in-loop + await sleepUnlessInfraDead(5000); + } + }; + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS the nodes upgrade from: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + nodeImageTags: { [STANDBY]: BASE_TAG, [PRIMARY]: BASE_TAG }, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + if (BASE_TAG === env.nodeImageTag(RING)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG: the suite upgrades from the one to the other'); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + expect(`${client(STANDBY).ip}:16127` < `${client(PRIMARY).ip}:16127`, 'fixture: the standby sorts before the primary by address').to.equal(true); + + await pushTestApp(appName, 'v1', 'roll', { user: APP_UID }); + const app = await buildSeedableApp({ + env, + name: appName, + instances: HOLDERS.length, + compose: COMPONENTS.map((name) => ({ + name, + description: `${name} of a two-writer app`, + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], + commands: [], + containerPorts: [80], + containerData: 'g:/appdata', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + })), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs only the first g: component on the release', async function () { + this.timeout(180000); + await holdsFor(60000, { [STANDBY]: [], [PRIMARY]: ['db'] }); + }); + + it('starts nothing on the upgraded standby while the primary on the release holds the app', async function () { + this.timeout(420000); + await env.upgradeNode(STANDBY, { tag: env.nodeImageTag(RING) }); + await holdsFor(90000, { [STANDBY]: [], [PRIMARY]: ['db'] }); + }); + + it('starts every g: component on the primary once it is upgraded', async function () { + this.timeout(600000); + await env.upgradeNode(PRIMARY, { tag: env.nodeImageTag(RING) }); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 420000, interval: 3000, label: 'the primary runs db and web', + }); + expect((await getFolderConfig(client(PRIMARY), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitHolding(async () => { + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + return isFolderSynced(client(STANDBY), folderOf('web')); + }, { timeout: 300000, interval: 3000, label: 'the standby has the primary\'s web data' }); + expect((await getFolderConfig(client(STANDBY), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + }); +}); diff --git a/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js b/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js new file mode 100644 index 0000000000..afac116d0d --- /dev/null +++ b/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js @@ -0,0 +1,151 @@ +// base-image: FLUX_E2E_BASE_TAG +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { listAppContainers } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, held by two nodes on a release +// that starts only the first, upgraded to this tree one node at a time: the +// primary first, then the standby. Real syncthing daemons. +// +// On the release the primary runs the first g: component and never the second, +// and the standby, whose address sorts first, takes the second component's empty +// folder as its own seed. Upgraded, the primary starts the second component beside +// the first while the standby is still on the release: the folder is seeded by the +// node that writes the app, whatever a standby on the release decided about it. +// Upgrading the standby changes nothing. At no point does any g: component of the +// app run on two nodes. +// +// Node 2 runs this tree throughout and holds nothing: the third node of the ring. +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + +const APP_UID = 1000; +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; + +describe('a rolling upgrade, primary first, brings up every g: component', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2erollp${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const folderOf = (name) => `flux${name}_${appName}`; + const STANDBY = 0; + const PRIMARY = 1; + const RING = 2; + const HOLDERS = [STANDBY, PRIMARY]; + + const client = (i) => env.clients[i]; + const runningOn = async (i) => { + const containers = await listAppContainers(client(i).container); + return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); + }; + // Which holders run any of the app's g: components; asserted on every poll. + const oneHolderAtMost = async () => { + const running = await Promise.all(HOLDERS.map(runningOn)); + const holders = HOLDERS.filter((_, k) => running[k].length); + expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); + return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); + }; + // Held over a minute of election passes, not read once. + const holdsFor = async (ms, expected) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + const running = await oneHolderAtMost(); + expect(running, 'what each holder runs').to.deep.equal(expected); + // eslint-disable-next-line no-await-in-loop + await sleepUnlessInfraDead(5000); + } + }; + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS the nodes upgrade from: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + nodeImageTags: { [STANDBY]: BASE_TAG, [PRIMARY]: BASE_TAG }, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + if (BASE_TAG === env.nodeImageTag(RING)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG: the suite upgrades from the one to the other'); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + expect(`${client(STANDBY).ip}:16127` < `${client(PRIMARY).ip}:16127`, 'fixture: the standby sorts before the primary by address').to.equal(true); + + await pushTestApp(appName, 'v1', 'roll', { user: APP_UID }); + const app = await buildSeedableApp({ + env, + name: appName, + instances: HOLDERS.length, + compose: COMPONENTS.map((name) => ({ + name, + description: `${name} of a two-writer app`, + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], + commands: [], + containerPorts: [80], + containerData: 'g:/appdata', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + })), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs only the first g: component on the release', async function () { + this.timeout(180000); + await holdsFor(60000, { [STANDBY]: [], [PRIMARY]: ['db'] }); + }); + + it('starts every g: component on the primary once it is upgraded, with the standby on the release', async function () { + this.timeout(600000); + await env.upgradeNode(PRIMARY, { tag: env.nodeImageTag(RING) }); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 420000, interval: 3000, label: 'the primary runs db and web', + }); + expect((await getFolderConfig(client(PRIMARY), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitHolding(async () => { + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + return isFolderSynced(client(STANDBY), folderOf('web')); + }, { timeout: 300000, interval: 3000, label: 'the standby on the release has the primary\'s web data' }); + }); + + it('changes nothing when the standby is upgraded', async function () { + this.timeout(420000); + await env.upgradeNode(STANDBY, { tag: env.nodeImageTag(RING) }); + await holdsFor(90000, { [STANDBY]: [], [PRIMARY]: COMPONENTS }); + expect((await getFolderConfig(client(STANDBY), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + }); +}); From 719b3b0ba8cc50a02cadcbe347aa17c52fd8fcc4 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 14:10:08 +0100 Subject: [PATCH 020/369] fix(syncthing): the writer of an app seeds a new g: folder over a standby's unsent files A standby's bytes in a g: folder that it never sent are a local change against the app's primary, which a standby's copy of any g: folder has reverted. The writer of the app's other g: components no longer stands aside for them: it seeds the folder when no peer serves it, and the standby's copy is reverted to it. Co-Authored-By: Claude Opus 5.5 --- .../syncthingFolderStateMachine.js | 10 ++++------ .../unit/syncthingFolderStateMachine.test.js | 20 ++++++++++++++++++- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index f762d27908..fb9aedc333 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1459,8 +1459,6 @@ async function handleReceiveOnlyTransition(params) { // its address differ, and the ranking degrades to the address order silently. const ownClaim = electedAppClaim(holdings, electedSiblings.map((sibling) => globalState.folderHoldings?.get(sibling))); const claims = {}; - // Whether a peer says it holds bytes of this folder itself. - let aPeerHoldsThisFolder = false; await Promise.all(electionList.map(async (peer) => { if (socketAddressesMatch(peer.ip, localSocketAddr)) { if (ownClaim) claims[peer.ip] = ownClaim; @@ -1474,7 +1472,6 @@ async function handleReceiveOnlyTransition(params) { // an outcome. Deliberately untested for that reason - a test would pass whether // this condition were here or not. if (answer?.ready && answer.holding && answer.holding[appId]) { - if ((answer.holding[appId].bytes || 0) > 0) aPeerHoldsThisFolder = true; const claim = electedAppClaim(answer.holding[appId], electedSiblings.map((sibling) => answer.holding[sibling])); if (claim) claims[peer.ip] = claim; } @@ -1487,13 +1484,14 @@ async function handleReceiveOnlyTransition(params) { const someoneElseHolds = Object.entries(claims) .some(([ip, claim]) => !socketAddressesMatch(ip, localSocketAddr) && (claim?.bytes || 0) > 0); const standDownOnUnknown = ownHoldingsUnknown && someoneElseHolds; - // The writer of the app's other g: components seeds this folder when there is - // nothing to seed it from, and no other node does. + // The writer of the app's other g: components seeds this folder when no peer + // serves it, and no other node does. What a standby holds in it and never sent + // is a local change against the app's primary, reverted as any standby's is. const appWriter = electedSiblings.length ? await electedAppWriter(electedSiblings, electionList, localSocketAddr, liveness) : null; const electedLeader = appWriter - ? appWriter.here && !aPeerHasData && !aPeerHoldsThisFolder + ? appWriter.here && !aPeerHasData : !standDownOnUnknown && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); // The floor holderIsGone asks of a silent holder, asked of this node before // its own win can count: a node whose peers have gone quiet is the one that diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index e5e709f1a0..bd6ffdb2c5 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2461,11 +2461,29 @@ describe('syncthingFolderStateMachine tests', () => { expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); }); - it('is not seeded by the writer over a peer holding bytes of this folder', async () => { + // What a standby holds in the folder and never sent is a local change + // against the app's primary: the writer seeds over it, and the standby's + // copy is reverted as any standby's local change is. + it('is seeded by the writer over a standby holding bytes of this folder it never sent', async () => { freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 900, newestModified: 5 } } } }); appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + // Canary for the test above: a peer that serves the folder is a source. + // The writer's synced copy carrying local changes of its own is reverted + // to it, never published as the seed. + it('reverts the writer\'s own local changes to a peer that serves the folder, rather than seeding them', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + peerHoldsTheData(); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 4096, globalFiles: 1, inSyncBytes: 4096, state: 'idle', receiveOnlyChangedFiles: 1, + }); + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + sinon.assert.calledWith(syncthingServiceMock.dbRevert, 'test-app'); }); it('is left to a peer whose folder for the app\'s other g: component sends, though this node sorts first', async () => { From 871b6ebcde540aa19ce69a5e9509c786ab35d0b5 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 14:16:39 +0100 Subject: [PATCH 021/369] fix(syncthing): the writer of an app seeds a new g: folder over a peer's seed mark A node that does not write an app cannot act on a seed mark it holds for one of its g: folders: it never starts the component, so the folder never sends. A standby on a release that elects one g: component takes such a folder as its seed by the address order and never withdraws the mark. The writer of the app's other g: components now yields only to a peer whose folder already sends. Co-Authored-By: Claude Opus 5.5 --- .../syncthingFolderStateMachine.js | 12 +++++++++--- .../unit/syncthingFolderStateMachine.test.js | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index fb9aedc333..8bd82b9221 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -989,17 +989,23 @@ function seedClaimRanksFirst(a, b) { * @param {string} localSocketAddr This node's socket address * @param {Object} liveness This pass's peer view * @param {{address: string, bytes: *, newestModified: *}} own This node's claim + * @param {object} [options] + * @param {boolean} [options.writer] This node writes the app's other g: + * components. Only a peer whose folder already sends stops it then: a seed mark + * is an intent to seed, and a node that does not write the app cannot act on + * one - it never starts the component, so its folder never sends. * @returns {Promise<{ip: string, reason: string, outcome: string}|null>} */ -async function findSeedRival(appId, peers, localSocketAddr, liveness, own) { +async function findSeedRival(appId, peers, localSocketAddr, liveness, own, { writer = false } = {}) { const others = (peers || []).filter((peer) => peer?.ip && !socketAddressesMatch(peer.ip, localSocketAddr)); const answers = await Promise.all(others.map( async (peer) => ({ ip: peer.ip, ...await liveness.reread(peer.ip) }), )); const answered = answers.filter((answer) => answer.reachable && answer.answerable); - const decided = answered.find((answer) => answer.seeding?.[appId]?.stage === 'decided' + const decided = answered.find((answer) => (!writer && answer.seeding?.[appId]?.stage === 'decided') || (answer.ready && answer.folders.includes(appId))); if (decided) return { ip: decided.ip, reason: 'has already decided to seed it', outcome: 'yieldedToDecided' }; + if (writer) return null; const rival = answered.find((answer) => { const mark = answer.seeding?.[appId]; return (mark?.stage === 'intent' || mark?.stage === 'deciding') @@ -1564,7 +1570,7 @@ async function handleReceiveOnlyTransition(params) { }; await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_SEED_DECISION, appId); const decisionStartedAt = monotonicMs(); - const rival = await findSeedRival(appId, runningAppList, localSocketAddr, liveness, ownSeedClaim); + const rival = await findSeedRival(appId, runningAppList, localSocketAddr, liveness, ownSeedClaim, { writer: appWriter?.here === true }); if (rival) { fluxEventBus.count('syncthing:seedDecision', appId, rival.outcome); log.info(`handleReceiveOnlyTransition - ${appId} is elected to seed but ${rival.ip} ${rival.reason}; going on as a standby`); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index bd6ffdb2c5..db56a03a18 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2486,6 +2486,25 @@ describe('syncthingFolderStateMachine tests', () => { sinon.assert.calledWith(syncthingServiceMock.dbRevert, 'test-app'); }); + // A standby on a release that elects one g: component takes the empty + // folder as its seed by the address order, and never withdraws the mark. + // It never starts the component, so its folder never sends. + it('is seeded by the writer over a peer\'s seed mark for it', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { seeding: { 'test-app': { stage: 'decided', bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + // Canary for the test above: a peer whose folder already sends has seeded + // it, and the writer stands aside. + it('is not seeded by the writer over a peer whose folder for it already sends', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { folders: ['test-app'] } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + it('is left to a peer whose folder for the app\'s other g: component sends, though this node sorts first', async () => { freshFolder({ peerAnswer: { folders: [SIBLING] } }); appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); From cc686023f67060b74b329ab44e13e112ed6c709f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:24:49 +0100 Subject: [PATCH 022/369] test(harness): an app's g: components are watched through one module g-components.js builds a g: component of the test image and answers what each holder runs of an app's g: components, asserting on every read that no two nodes run any of them at once. 2241-2244 read it from there. syncthing-real.js can pause or resume one folder, as pauseAllFolders does folder by folder. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/g-components.js | 84 +++++++++++++++++++ test-infra/runner/framework/syncthing-real.js | 13 ++- ...-components-run-together-on-its-primary.js | 73 +++------------- ...-a-g-component-runs-the-app-on-one-node.js | 39 ++------- ...andby-first-brings-up-every-g-component.js | 46 ++-------- ...imary-first-brings-up-every-g-component.js | 46 ++-------- 6 files changed, 124 insertions(+), 177 deletions(-) create mode 100644 test-infra/runner/framework/g-components.js diff --git a/test-infra/runner/framework/g-components.js b/test-infra/runner/framework/g-components.js new file mode 100644 index 0000000000..736366e44a --- /dev/null +++ b/test-infra/runner/framework/g-components.js @@ -0,0 +1,84 @@ +import { expect } from 'chai'; +import { listAppContainers } from './container.js'; +import { REGISTRY_REPO_HOST } from './subnet-config.js'; +import { sleepUnlessInfraDead } from './infra-death.js'; + +// An app with more than one single-writer (g:) component, as the suites that watch +// its election see it. +// +// The election decides the app, not each component: its primary runs every g: +// component on one node, and every other holder runs none. A node running one of +// them while another node runs any other is the app split across two nodes - the +// load balancer sends all of its ports to one of them, and the components cannot +// reach each other over the app's network - so every wait built from this holds +// that no two nodes run any of its g: components at once. + +/** + * One g: component of the test image, which writes `.txt` into its volume. + * Ports are left to buildSeedableApp, which allocates them by compose index; a + * component added by an update takes its own (allocatePortFor(appName, index)). + * @param {string} appName + * @param {string} name + * @returns {object} a compose entry + */ +export function gComponent(appName, name) { + return { + name, + description: `${name} of an app with several writers`, + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], + commands: [], + containerPorts: [80], + containerData: 'g:/appdata', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +/** + * What each holder runs of an app's g: components. + * @param {() => object} getEnv the suite's env, read when called + * @param {object} opts + * @param {string} opts.appName + * @param {string[]} opts.components every g: component name the app has at any + * point of the suite, in the order results list them + * @param {number[]} opts.holders node indices that hold the app + * @returns {{ + * folderOf: (name: string) => string, + * runningOn: (i: number) => Promise, + * oneHolderAtMost: () => Promise>, + * holdsFor: (ms: number, expected: Object) => Promise, + * }} + */ +export function watchGComponents(getEnv, { appName, components, holders }) { + const folderOf = (name) => `flux${name}_${appName}`; + const runningOn = async (i) => { + const containers = await listAppContainers(getEnv().clients[i].container); + return components.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); + }; + // Asserted on every call: a waitHolding poll built on it fails at the first breach. + const oneHolderAtMost = async () => { + const running = await Promise.all(holders.map(runningOn)); + const runners = holders.filter((_, k) => running[k].length); + expect(runners.length, `the app runs on more than one node: ${holders.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); + return Object.fromEntries(holders.map((i, k) => [i, running[k]])); + }; + // Held over a span of election passes, not read once. + const holdsFor = async (ms, expected) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + const running = await oneHolderAtMost(); + expect(running, 'what each holder runs').to.deep.equal(expected); + // eslint-disable-next-line no-await-in-loop + await sleepUnlessInfraDead(5000); + } + }; + return { + folderOf, runningOn, oneHolderAtMost, holdsFor, + }; +} diff --git a/test-infra/runner/framework/syncthing-real.js b/test-infra/runner/framework/syncthing-real.js index 4831b2840b..b3b892b71e 100644 --- a/test-infra/runner/framework/syncthing-real.js +++ b/test-infra/runner/framework/syncthing-real.js @@ -1,7 +1,8 @@ // Reads a node's OWN syncthing daemon, for suites booted with // `createTestEnv({ syncthing: 'binary' })`. // -// Deliberately observers, not setters. The stub's control surface +// Observers, apart from the actions a machine itself takes on its daemon +// (start, stop, scan, pause a folder). The stub's control surface // (syncthing-control.js) scripts what a node believes about sync state, because // with a stub there is no truth to read. With a real daemon there is, and // scripting it would only hide what the test exists to establish - so these @@ -228,6 +229,16 @@ export async function stopDaemon(client) { await execInContainer(client.container, 'pkill -KILL -x syncthing --ns 1 --nslist pid; true'); } +// Pause or resume one folder, as FluxOS's pauseAllFolders does folder by folder - +// so a FluxOS that stops partway leaves some of a node's folders paused and the +// rest shared. Peers then read this node's connection to that folder as closed. +export async function setFolderPaused(client, folderId, paused) { + const key = await apiKey(client); + const r = await execInContainer(client.container, + `curl -sS -f -X PATCH -H "X-API-Key: ${key}" -H "Content-Type: application/json" -d '{"paused":${paused ? 'true' : 'false'}}' "http://127.0.0.1:8384/rest/config/folders/${encodeURIComponent(folderId)}"`); + if (r.exitCode !== 0) throw new Error(`syncthing-real: ${paused ? 'pause' : 'resume'} of ${folderId} failed: ${r.stderr || r.output}`); +} + // The daemon's own event log since `since`, optionally of named types - e.g. // StateChanged (a folder starting a scan) and ConfigSaved (a folder's type // changed). Event ids are ordered, so two events can be put in order, and diff --git a/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js b/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js index f6c0745ac3..d89c0c8b5a 100644 --- a/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js +++ b/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js @@ -1,9 +1,7 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { - execInContainer, listAppContainers, crashFluxos, releaseFluxos, -} from '../framework/container.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; @@ -11,18 +9,12 @@ import { waitFor, waitHolding } from '../framework/wait.js'; import { isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, } from '../framework/syncthing-real.js'; -import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; -// An app with two single-writer (g:) components, on real syncthing daemons. -// -// The election decides the app, not each component: its primary runs every g: -// component on one node, and every other holder runs none. A node running one of -// them and not the other is the app split across two nodes - the load balancer -// sends all of its ports to one of them, and the components cannot reach each -// other over the app's network - so every wait below holds that no two nodes run -// any of its g: components at once. +// An app with two single-writer (g:) components, on real syncthing daemons: its +// primary runs both, and every other holder runs neither. Every wait holds that no +// two nodes run any of its g: components at once (g-components.js). const APP_UID = 1000; @@ -32,7 +24,6 @@ describe('an app\'s g: components run together on its primary', function () { const appName = `e2etwin${Date.now()}`; const COMPONENTS = ['db', 'web']; - const folderOf = (component) => `flux${component}_${appName}`; // Placed in this order, so the election ranks them in it; a third node holds // nothing and keeps the ring whole. const FIRST = 0; @@ -40,20 +31,9 @@ describe('an app\'s g: components run together on its primary', function () { const HOLDERS = [FIRST, SECOND]; const client = (i) => env.clients[i]; - // The app's g: components running on a node, in compose order. - const runningOn = async (i) => { - const containers = await listAppContainers(client(i).container); - return COMPONENTS.filter((component) => containers.some( - (c) => c.name === `flux${component}_${appName}` && c.status?.startsWith('Up'), - )); - }; - // Which holders run any of the app's g: components; asserted on every poll. - const oneHolderAtMost = async () => { - const running = await Promise.all(HOLDERS.map(runningOn)); - const holders = HOLDERS.filter((_, k) => running[k].length); - expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); - return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); - }; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (component) => (await getFolderConfig(client(i), folderOf(component)))?.type)); before(async function () { @@ -77,21 +57,7 @@ describe('an app\'s g: components run together on its primary', function () { env, name: appName, instances: HOLDERS.length, - compose: COMPONENTS.map((component) => ({ - name: component, - description: `${component} of a two-writer app`, - repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, - ports: [], - domains: [''], - environmentParameters: [`WRITE_FILE=/appdata/${component}.txt`, `WRITE_CONTENT=${component}`], - commands: [], - containerPorts: [80], - containerData: 'g:/appdata', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', - })), + compose: COMPONENTS.map((name) => gComponent(appName, name)), }); await installOnNodes(env, app, [FIRST]); @@ -111,16 +77,7 @@ describe('an app\'s g: components run together on its primary', function () { it('runs every g: component on the primary, and none on the standby', async function () { this.timeout(180000); - // Held over several election passes, not read once. - const deadline = Date.now() + 60000; - while (Date.now() < deadline) { - // eslint-disable-next-line no-await-in-loop - const running = await oneHolderAtMost(); - expect(running[FIRST], 'the primary').to.deep.equal(COMPONENTS); - expect(running[SECOND], 'the standby').to.deep.equal([]); - // eslint-disable-next-line no-await-in-loop - await sleepUnlessInfraDead(5000); - } + await holdsFor(60000, { [FIRST]: COMPONENTS, [SECOND]: [] }); expect(await folderTypes(FIRST), 'the primary\'s folders').to.deep.equal(['sendreceive', 'sendreceive']); expect(await folderTypes(SECOND), 'the standby\'s folders').to.deep.equal(['receiveonly', 'receiveonly']); }); @@ -148,14 +105,6 @@ describe('an app\'s g: components run together on its primary', function () { await waitFor(async () => (await folderTypes(FIRST)).every((type) => type === 'receiveonly'), { timeout: 300000, interval: 3000, label: 'every folder of the old primary receives', }); - const deadline = Date.now() + 60000; - while (Date.now() < deadline) { - // eslint-disable-next-line no-await-in-loop - const running = await oneHolderAtMost(); - expect(running[FIRST], 'the old primary').to.deep.equal([]); - expect(running[SECOND], 'the primary').to.deep.equal(COMPONENTS); - // eslint-disable-next-line no-await-in-loop - await sleepUnlessInfraDead(5000); - } + await holdsFor(60000, { [FIRST]: [], [SECOND]: COMPONENTS }); }); }); diff --git a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js index 367662ce72..700a8f7e01 100644 --- a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js +++ b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js @@ -2,7 +2,6 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { listAppContainers } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; import { allocatePortFor } from '../framework/port-allocator.js'; @@ -10,7 +9,7 @@ import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconc import { waitFor, waitHolding } from '../framework/wait.js'; import { driveUntil, stopTicker } from '../framework/daemon-control.js'; import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; -import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // A specification update gives a running app a second single-writer (g:) @@ -20,8 +19,8 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // and installs it at the new specification, on its own reinstall pass. Between the // two passes one holder is at the new specification and the other still runs the // old one, and after both the app starts afresh. Through all of it no two nodes run -// any g: component of the app at once, and once settled one node runs both and the -// other runs neither. +// any g: component of the app at once (g-components.js), and once settled one node +// runs both and the other runs neither. // // DRIVEN BY BLOCKS: the reinstall pass fires on // `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). @@ -34,42 +33,18 @@ describe('an update that adds a g: component runs the app on one node', function const appName = `e2eaddg${Date.now()}`; const COMPONENTS = ['db', 'web']; - const folderOf = (name) => `flux${name}_${appName}`; const HOLDERS = [0, 1]; let app; const client = (i) => env.clients[i]; - const component = (name) => ({ - name, - description: `${name} of an app that gains a writer`, - repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, - ports: [], - domains: [''], - environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], - commands: [], - containerPorts: [80], - containerData: 'g:/appdata', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, }); // The specification a holder has installed, by its hash. const installedHash = async (i) => { const res = await client(i).getInstalledApps(); return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; }; - const runningOn = async (i) => { - const containers = await listAppContainers(client(i).container); - return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); - }; - // Which holders run any of the app's g: components; asserted on every poll. - const oneHolderAtMost = async () => { - const running = await Promise.all(HOLDERS.map(runningOn)); - const holders = HOLDERS.filter((_, k) => running[k].length); - expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); - return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); - }; before(async function () { this.timeout(900000); @@ -89,7 +64,7 @@ describe('an update that adds a g: component runs the app on one node', function await pushTestApp(appName, 'v1', 'addg', { user: APP_UID }); app = await buildSeedableApp({ - env, name: appName, instances: HOLDERS.length, compose: [component('db')], + env, name: appName, instances: HOLDERS.length, compose: [gComponent(appName, 'db')], }); await installOnNodes(env, app, [HOLDERS[0]]); await waitHolding(async () => (await oneHolderAtMost())[HOLDERS[0]].includes('db'), { @@ -111,7 +86,7 @@ describe('an update that adds a g: component runs the app on one node', function this.timeout(1800000); const updated = await buildSeedableUpdate(app, (spec) => { // The new component's port, allocated as registration allocates db's. - spec.compose.push({ ...component('web'), ports: [allocatePortFor(appName, 1)] }); + spec.compose.push({ ...gComponent(appName, 'web'), ports: [allocatePortFor(appName, 1)] }); }); await seedSpecUpdate(env, updated, HOLDERS); await stopTicker(); diff --git a/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js b/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js index f94d397124..793c9c4f87 100644 --- a/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js +++ b/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js @@ -2,14 +2,12 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { listAppContainers } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; import { waitFor, waitHolding } from '../framework/wait.js'; import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; -import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // An app with two single-writer (g:) components, held by two nodes on a release @@ -36,35 +34,15 @@ describe('a rolling upgrade, standby first, brings up every g: component', funct const appName = `e2eroll${Date.now()}`; const COMPONENTS = ['db', 'web']; - const folderOf = (name) => `flux${name}_${appName}`; const STANDBY = 0; const PRIMARY = 1; const RING = 2; const HOLDERS = [STANDBY, PRIMARY]; const client = (i) => env.clients[i]; - const runningOn = async (i) => { - const containers = await listAppContainers(client(i).container); - return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); - }; - // Which holders run any of the app's g: components; asserted on every poll. - const oneHolderAtMost = async () => { - const running = await Promise.all(HOLDERS.map(runningOn)); - const holders = HOLDERS.filter((_, k) => running[k].length); - expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); - return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); - }; - // Held over a minute of election passes, not read once. - const holdsFor = async (ms, expected) => { - const deadline = Date.now() + ms; - while (Date.now() < deadline) { - // eslint-disable-next-line no-await-in-loop - const running = await oneHolderAtMost(); - expect(running, 'what each holder runs').to.deep.equal(expected); - // eslint-disable-next-line no-await-in-loop - await sleepUnlessInfraDead(5000); - } - }; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); before(async function () { this.timeout(900000); @@ -93,21 +71,7 @@ describe('a rolling upgrade, standby first, brings up every g: component', funct env, name: appName, instances: HOLDERS.length, - compose: COMPONENTS.map((name) => ({ - name, - description: `${name} of a two-writer app`, - repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, - ports: [], - domains: [''], - environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], - commands: [], - containerPorts: [80], - containerData: 'g:/appdata', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', - })), + compose: COMPONENTS.map((name) => gComponent(appName, name)), }); await installOnNodes(env, app, [PRIMARY]); await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { diff --git a/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js b/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js index afac116d0d..2d9843495a 100644 --- a/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js +++ b/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js @@ -2,14 +2,12 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { listAppContainers } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; import { waitFor, waitHolding } from '../framework/wait.js'; import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; -import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // An app with two single-writer (g:) components, held by two nodes on a release @@ -36,35 +34,15 @@ describe('a rolling upgrade, primary first, brings up every g: component', funct const appName = `e2erollp${Date.now()}`; const COMPONENTS = ['db', 'web']; - const folderOf = (name) => `flux${name}_${appName}`; const STANDBY = 0; const PRIMARY = 1; const RING = 2; const HOLDERS = [STANDBY, PRIMARY]; const client = (i) => env.clients[i]; - const runningOn = async (i) => { - const containers = await listAppContainers(client(i).container); - return COMPONENTS.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); - }; - // Which holders run any of the app's g: components; asserted on every poll. - const oneHolderAtMost = async () => { - const running = await Promise.all(HOLDERS.map(runningOn)); - const holders = HOLDERS.filter((_, k) => running[k].length); - expect(holders.length, `the app runs on more than one node: ${HOLDERS.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); - return Object.fromEntries(HOLDERS.map((i, k) => [i, running[k]])); - }; - // Held over a minute of election passes, not read once. - const holdsFor = async (ms, expected) => { - const deadline = Date.now() + ms; - while (Date.now() < deadline) { - // eslint-disable-next-line no-await-in-loop - const running = await oneHolderAtMost(); - expect(running, 'what each holder runs').to.deep.equal(expected); - // eslint-disable-next-line no-await-in-loop - await sleepUnlessInfraDead(5000); - } - }; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); before(async function () { this.timeout(900000); @@ -93,21 +71,7 @@ describe('a rolling upgrade, primary first, brings up every g: component', funct env, name: appName, instances: HOLDERS.length, - compose: COMPONENTS.map((name) => ({ - name, - description: `${name} of a two-writer app`, - repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, - ports: [], - domains: [''], - environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], - commands: [], - containerPorts: [80], - containerData: 'g:/appdata', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', - })), + compose: COMPONENTS.map((name) => gComponent(appName, name)), }); await installOnNodes(env, app, [PRIMARY]); await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { From 7f1d4a579cd13f4dc85a374555c1e4b1bb838001 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:25:35 +0100 Subject: [PATCH 023/369] test(harness): 2245 restarts a g: component that dies on its primary there A g: component of a two-writer app is killed on its primary while its FluxOS, syncthing and the other component run on: the primary starts it again, and no other node starts any of the app's g: components meanwhile. The second, the first, and both. Co-Authored-By: Claude Opus 5.5 --- ...-dies-on-its-primary-is-restarted-there.js | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js diff --git a/test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js b/test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js new file mode 100644 index 0000000000..a4e5b4b8d6 --- /dev/null +++ b/test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js @@ -0,0 +1,111 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, on real syncthing daemons. One of +// them dies on the primary while its FluxOS, its syncthing and the other component +// go on running. +// +// The primary still holds the app: it runs the other component, and its folders +// still send. It starts the dead one again where it was. A standby that started it +// would run the app on two nodes - the one it started, beside the one the primary +// still runs - so no other node starts any of its g: components meanwhile. + +const APP_UID = 1000; + +describe('a g: component that dies on its primary is restarted there', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egdies${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // Placed in this order, so the election ranks them in it; a third node holds + // nothing and keeps the ring whole. + const PRIMARY = 0; + const STANDBY = 1; + const HOLDERS = [PRIMARY, STANDBY]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + // When docker last started a component's container on a node. + const startedAt = async (i, name) => { + const r = await execInContainer(client(i).container, `docker inspect -f '{{.State.StartedAt}}' ${folderOf(name)}`); + if (r.exitCode !== 0 || !r.stdout.trim()) throw new Error(`docker inspect ${folderOf(name)} on node ${i} failed: ${r.stderr || r.stdout}`); + return r.stdout.trim(); + }; + // Kills the given components on the primary, and holds until it runs every g: + // component again, each killed one in a container docker started after the kill. + const restartedInPlace = async (names, label) => { + const before = await Promise.all(names.map((name) => startedAt(PRIMARY, name))); + await Promise.all(names.map((name) => execInContainer(client(PRIMARY).container, `docker kill ${folderOf(name)}`))); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + if (running[PRIMARY].length !== COMPONENTS.length) return false; + const after = await Promise.all(names.map((name) => startedAt(PRIMARY, name))); + return after.every((at, k) => at !== before[k]); + }, { timeout: 300000, interval: 2000, label: `${label}: the primary runs every g: component again, the killed ones restarted` }); + await holdsFor(60000, { [PRIMARY]: COMPONENTS, [STANDBY]: [] }); + expect(await Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(PRIMARY), folderOf(name)))?.type)), 'the primary\'s folders') + .to.deep.equal(['sendreceive', 'sendreceive']); + }; + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gdies', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the primary runs every g: component', + }); + await installOnNodes(env, app, [STANDBY]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(STANDBY), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the standby has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('restarts the second g: component on the primary when it dies there', async function () { + this.timeout(600000); + await restartedInPlace(['web'], 'web killed'); + }); + + it('restarts the first g: component on the primary when it dies there', async function () { + this.timeout(600000); + await restartedInPlace(['db'], 'db killed'); + }); + + it('restarts both on the primary when both die there and its FluxOS goes on running', async function () { + this.timeout(600000); + await restartedInPlace(COMPONENTS, 'db and web killed'); + }); +}); From 9cc47b88c51b8bb26ee7bd3b7127a116ea5cdf6f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:26:55 +0100 Subject: [PATCH 024/369] test(harness): 2246 starts nothing beside a silent primary while any of its folders is connected A two-writer app's primary goes silent with its containers and syncthing up and one of the app's folders paused: the standby judges it alive by the folder still connected, pass after pass, and starts nothing. With the whole machine down it takes the app over. syncthing-real.js reads another device's side of a folder (getRemoteState). Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/syncthing-real.js | 7 + ...ed-beside-while-any-folder-is-connected.js | 121 ++++++++++++++++++ 2 files changed, 128 insertions(+) create mode 100644 test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js diff --git a/test-infra/runner/framework/syncthing-real.js b/test-infra/runner/framework/syncthing-real.js index b3b892b71e..ab2fae334a 100644 --- a/test-infra/runner/framework/syncthing-real.js +++ b/test-infra/runner/framework/syncthing-real.js @@ -62,6 +62,13 @@ export async function getFolderStatus(client, folderId) { return api(client, `/rest/db/status?folder=${encodeURIComponent(folderId)}`); } +// How this node's syncthing sees another device's side of a folder: 'valid' while +// that device shares it over a live connection, 'paused', 'notSharing' or +// 'unknown' otherwise. What FluxOS reads a peer's connection to the folder from. +export async function getRemoteState(client, folderId, deviceId) { + return (await api(client, `/rest/db/completion?folder=${encodeURIComponent(folderId)}&device=${encodeURIComponent(deviceId)}`)).remoteState; +} + // A folder is complete when the index describes something and the disk holds all // of it. globalBytes === 0 means the index is empty, which is not the same as // synced and must never read as done. diff --git a/test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js b/test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js new file mode 100644 index 0000000000..456b3f434a --- /dev/null +++ b/test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js @@ -0,0 +1,121 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, getDeviceId, getRemoteState, setFolderPaused, stopDaemon, +} from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, on real syncthing daemons. Its +// primary's FluxOS stops answering while its containers and its syncthing go on +// running, with one of the app's folders paused there and the other still shared. +// +// A standby reads a silent primary as gone only when its own syncthing shows the +// primary's connection gone for EVERY folder of the app. A folder still connected +// is a component the primary may still be writing, so the standby starts nothing: +// started, the app would run on both nodes. Once the whole machine is down - every +// folder's connection gone - the standby takes the app over. + +const APP_UID = 1000; +const HELD_PASSES = 3; + +describe('a silent primary is not started beside while any of its folders is connected', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egsilent${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // The anchor: the first g: component, which keys the app's election records. + const identifier = `db_${appName}`; + const PRIMARY = 0; + const STANDBY = 1; + const HOLDERS = [PRIMARY, STANDBY]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const silenceVerdicts = (verdict) => client(STANDBY).getDecisionCount('peer:silenceVerdict', identifier, verdict); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gsilent', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the primary runs every g: component', + }); + await installOnNodes(env, app, [STANDBY]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(STANDBY), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the standby has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(120000); + await releaseFluxos(client(PRIMARY).container).catch(() => {}); + await env?.teardown(); + }); + + it('starts nothing beside a silent primary whose second folder is still connected', async function () { + this.timeout(600000); + await crashFluxos(client(PRIMARY).container, { hold: true }); + await setFolderPaused(client(PRIMARY), folderOf('db'), true); + + const primaryDevice = await getDeviceId(client(PRIMARY)); + await waitFor(async () => (await getRemoteState(client(STANDBY), folderOf('db'), primaryDevice)) !== 'valid', { + timeout: 120000, interval: 2000, label: 'the standby\'s syncthing no longer sees the primary sharing db', + }); + expect(await getRemoteState(client(STANDBY), folderOf('web'), primaryDevice), 'fixture: the standby\'s syncthing sees the primary sharing web').to.equal('valid'); + + // Each is a pass in which the standby asked the silent primary and judged its + // silence: alive, by the folder still connected. + const gone = await silenceVerdicts('gone'); + const from = await silenceVerdicts('connectionAlive'); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + expect(running[PRIMARY], 'the silent primary').to.deep.equal(COMPONENTS); + return await silenceVerdicts('connectionAlive') >= from + HELD_PASSES; + }, { timeout: 300000, interval: 3000, label: `${HELD_PASSES} passes in which the standby judged the silent primary alive` }); + expect(await silenceVerdicts('gone') - gone, 'passes that read the silent primary as gone').to.equal(0); + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + }); + + it('takes the app over once the primary\'s whole machine is down', async function () { + this.timeout(600000); + const gone = await silenceVerdicts('gone'); + await Promise.all(COMPONENTS.map((name) => execInContainer(client(PRIMARY).container, `docker kill ${folderOf(name)}`))); + await stopDaemon(client(PRIMARY)); + + await waitHolding(async () => (await oneHolderAtMost())[STANDBY].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the standby runs every g: component', + }); + expect(await silenceVerdicts('gone') - gone, 'passes that read the primary as gone').to.be.above(0); + await waitFor(async () => (await Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(STANDBY), folderOf(name)))?.type))) + .every((type) => type === 'sendreceive'), { + timeout: 120000, interval: 2000, label: 'every folder of the new primary sends', + }); + }); +}); From 17423f083ea2f8355f08d57ff3726616f2524800 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:27:42 +0100 Subject: [PATCH 025/369] test(harness): 2247 an owner stop and start keeps every g: component on the primary A two-writer app's primary ranked below another holder: stopped and started everywhere, every g: component comes back on the primary. One component stopped alone on the primary - the second, and the first - stays down there while the primary runs the other and no other node starts either; started again, it comes back on the primary. Co-Authored-By: Claude Opus 5.5 --- ...-keeps-every-g-component-on-the-primary.js | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js diff --git a/test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js b/test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js new file mode 100644 index 0000000000..f107c1aa1e --- /dev/null +++ b/test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js @@ -0,0 +1,179 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, electionIndexOf } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding, electionDecisionCount } from '../framework/wait.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, +} from '../framework/syncthing-real.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An owner stops and starts an app with two single-writer (g:) components, on real +// syncthing daemons: the whole app on every node at once, and one component alone +// on its primary. +// +// Stopped and started everywhere, the app comes back on the node that was its +// primary, every g: component of it, not on whichever node the election order +// would pick: the primary is put below index 0 first, as in suite 2215. One +// component stopped on the primary stays down by its lock while the primary goes +// on running the other, and no other node starts either - the primary still holds +// the app. Started again, it comes back on the primary. + +const APP_UID = 1000; +const HELD_PASSES = 3; + +describe('an owner stop and start keeps every g: component on the primary', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egowner${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // The anchor: the first g: component, which keys the app's election records. + const identifier = `db_${appName}`; + // Placed in this order, so the election ranks them in it. A fourth node holds + // nothing and keeps the discovery ring whole. + const FIRST = 0; + const HOLDERS = [0, 1, 2]; + let PRIMARY; + let STANDBYS; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const runners = async () => Object.entries(await oneHolderAtMost()) + .filter(([, running]) => running.length).map(([i]) => Number(i)); + const ownerAuth = async (i) => (await authenticate(client(i).url, appOwnerKey())).zelidauth; + const owner = async (command, target, via, { global = false } = {}) => { + const res = await client(via).getAuthed(`/apps/${command}/${target}${global ? '/true' : ''}`, await ownerAuth(via)); + expect(res?.status, `owner ${global ? 'global ' : ''}${command} of ${target} through node ${via}: ${JSON.stringify(res)}`).to.equal('success'); + }; + const evaluated = (nodes) => Promise.all(nodes.map((i) => electionDecisionCount(client(i), identifier, 'evaluated'))); + const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(i), folderOf(name)))?.type)); + + // What every holder runs, asserted on every poll, until each of `nodes` has run + // HELD_PASSES election passes over the app. + const holdsThroughPasses = async (nodes, expected, label) => { + const from = await evaluated(nodes); + await waitHolding(async () => { + expect(await oneHolderAtMost(), `${label}: what each holder runs`).to.deep.equal(expected); + const now = await evaluated(nodes); + return now.every((n, k) => n >= from[k] + HELD_PASSES); + }, { timeout: 300000, interval: 2000, label: `${label}: ${HELD_PASSES} election passes on nodes ${nodes.join(', ')}` }); + }; + const nothingRuns = () => Object.fromEntries(HOLDERS.map((i) => [i, []])); + const onPrimary = (running) => ({ ...nothingRuns(), [PRIMARY]: running }); + + before(async function () { + this.timeout(1500000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gowner', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + + // One at a time: a node learns of the app only when it is installed, so the + // order is the placement order. + await installOnNodes(env, app, [FIRST]); + await waitHolding(async () => (await oneHolderAtMost())[FIRST].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + for (const i of HOLDERS.filter((h) => h !== FIRST)) { + // eslint-disable-next-line no-await-in-loop + await installOnNodes(env, app, [i]); + // eslint-disable-next-line no-await-in-loop + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(i), folderOf(name)), { + timeout: 300000, interval: 3000, label: `holder ${i} has the first holder's ${name} data`, + }))); + } + + // The first holder dies whole - process, containers and daemon - and another + // takes the app over; then it comes back, as a standby still ranked ahead of + // the primary. + await crashFluxos(client(FIRST).container, { hold: true }); + await Promise.all(COMPONENTS.map((name) => execInContainer(client(FIRST).container, `docker kill ${folderOf(name)}`))); + await stopDaemon(client(FIRST)); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + return HOLDERS.some((i) => i !== FIRST && running[i].length === COMPONENTS.length); + }, { timeout: 300000, interval: 3000, label: 'another holder runs every g: component' }); + [PRIMARY] = await runners(); + STANDBYS = HOLDERS.filter((i) => i !== PRIMARY); + + await startDaemon(client(FIRST)); + await releaseFluxos(client(FIRST).container); + await waitHolding(async () => { + await oneHolderAtMost(); + return (await folderTypes(FIRST)).every((type) => type === 'receiveonly'); + }, { timeout: 300000, interval: 2000, label: 'the first holder is back as a standby of the whole app' }); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(FIRST), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the first holder has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs the app on a primary the election order ranks below another holder', async function () { + this.timeout(180000); + await holdsFor(30000, onPrimary(COMPONENTS)); + expect(await electionIndexOf(env, appName, FIRST), 'the first holder\'s place').to.equal(0); + expect(await electionIndexOf(env, appName, PRIMARY), 'the primary\'s place').to.be.above(0); + }); + + it('brings every g: component back on its primary when its owner stops the app everywhere and starts it everywhere', async function () { + this.timeout(900000); + await owner('appstop', appName, PRIMARY, { global: true }); + await waitHolding(async () => (await runners()).length === 0, { + timeout: 180000, interval: 2000, label: 'the app stops everywhere', + }); + await holdsThroughPasses(STANDBYS, nothingRuns(), 'stopped everywhere'); + + await owner('appstart', appName, FIRST, { global: true }); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 2000, label: 'started everywhere: the primary runs every g: component again', + }); + await holdsThroughPasses(STANDBYS, onPrimary(COMPONENTS), 'started everywhere'); + expect(await folderTypes(PRIMARY), 'the primary\'s folders').to.deep.equal(['sendreceive', 'sendreceive']); + }); + + for (const [stopped, kept] of [['web', 'db'], ['db', 'web']]) { + it(`keeps ${kept} on the primary and runs ${stopped} nowhere when its owner stops ${stopped} alone there, and brings it back there once started`, async function () { + this.timeout(900000); + await owner('appstop', `${stopped}_${appName}`, PRIMARY); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + return running[PRIMARY].length === 1 && running[PRIMARY][0] === kept; + }, { timeout: 180000, interval: 2000, label: `${stopped} stops on the primary` }); + await holdsThroughPasses(HOLDERS, onPrimary([kept]), `${stopped} stopped on the primary`); + + await owner('appstart', `${stopped}_${appName}`, PRIMARY); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 2000, label: `${stopped} started on the primary: it runs every g: component again`, + }); + await holdsThroughPasses(STANDBYS, onPrimary(COMPONENTS), `${stopped} started again`); + }); + } +}); From 743ac82a8dd0f5131e33545359d813e1571d1d2d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:28:17 +0100 Subject: [PATCH 026/369] test(harness): 2248 an update that moves the first g: component runs the app on one node Two updates to a running two-writer app, each moving the g: component that names the app to the election: one reverses the components, the next removes the one that became first. Through each holder's reinstall no two nodes run any g: component of the app, and once both are at the update one node runs it. Co-Authored-By: Claude Opus 5.5 --- ...st-g-component-runs-the-app-on-one-node.js | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js diff --git a/test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js b/test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js new file mode 100644 index 0000000000..c040091e01 --- /dev/null +++ b/test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js @@ -0,0 +1,129 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Specification updates move the first single-writer (g:) component of a running +// app with two, on real syncthing daemons: one reverses the components, the next +// removes the one that became first. +// +// The first g: component names the app to the election - it keys the election's +// records, and it is the component a node on a release asks a peer about - so each +// update moves the app's election onto another component. Each holder reinstalls +// on its own pass (suite 2242); through all of it no two nodes run any g: +// component of the app at once (g-components.js), and once settled one node runs +// every g: component the app has and the other runs none. +// +// DRIVEN BY BLOCKS: the reinstall pass fires on +// `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). + +const APP_UID = 1000; + +describe('an update that moves the first g: component runs the app on one node', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2emoveg${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const HOLDERS = [0, 1]; + let app; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + // The specification a holder has installed, by its hash. + const installedHash = async (i) => { + const res = await client(i).getInstalledApps(); + return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; + }; + + // Seeds the update and drives blocks until both holders are at it and one runs + // exactly `expected`, then holds that. Returns the node that runs it. + const settlesOnOneNode = async (updated, expected) => { + await seedSpecUpdate(env, updated, HOLDERS); + await stopTicker(); + + let breach = null; + let settled = null; + await driveUntil(client(HOLDERS[0]), async () => { + try { + const running = await oneHolderAtMost(); + settled = HOLDERS.find((i) => running[i].length) ?? null; + const hashes = await Promise.all(HOLDERS.map(installedHash)); + return settled !== null && JSON.stringify(running[settled]) === JSON.stringify(expected) + && hashes.every((hash) => hash === updated.hash); + } catch (error) { + if (error?.name !== 'AssertionError') throw error; + breach = error; + return true; + } + }, { blocks: 216, label: `both holders are at the new specification and one runs [${expected}]` }); + if (breach) throw breach; + + const other = HOLDERS.find((i) => i !== settled); + await holdsFor(60000, { [settled]: expected, [other]: [] }); + expect(await Promise.all(expected.map(async (name) => (await getFolderConfig(client(settled), folderOf(name)))?.type)), 'the primary\'s folders') + .to.deep.equal(expected.map(() => 'sendreceive')); + return settled; + }; + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'moveg', { user: APP_UID }); + app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [HOLDERS[0]]); + await waitHolding(async () => (await oneHolderAtMost())[HOLDERS[0]].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + await installOnNodes(env, app, [HOLDERS[1]]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(HOLDERS[1]), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the second holder has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs the app on one node once an update puts its second g: component first', async function () { + // A backstop: the block drive bounds this test (suite 85). + this.timeout(1800000); + // Each component keeps the port it was registered with. + app = await buildSeedableUpdate(app, (spec) => { spec.compose.reverse(); }); + expect(app.spec.compose.map((c) => c.name), 'fixture: the update\'s components').to.deep.equal(['web', 'db']); + await settlesOnOneNode(app, COMPONENTS); + }); + + it('runs the app on one node once an update removes the g: component that is first', async function () { + this.timeout(1800000); + app = await buildSeedableUpdate(app, (spec) => { spec.compose = spec.compose.filter((c) => c.name !== 'web'); }); + expect(app.spec.compose.map((c) => c.name), 'fixture: the update\'s components').to.deep.equal(['db']); + await settlesOnOneNode(app, ['db']); + }); +}); From e49384684613757490d1b1da2aeb9da5ee82bcdc Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:30:03 +0100 Subject: [PATCH 027/369] test(harness): a cold start and a holder's data are pinned through syncthing-control pinColdStart and pinHolding declare a stub folder's cold-start shape and one holder's data; 51 and 2214 read them from there. Co-Authored-By: Claude Opus 5.5 --- .../runner/framework/syncthing-control.js | 39 ++++++++++++++++++ ...ers-that-see-each-other-never-both-seed.js | 33 +-------------- .../51-reconciler-syncthing-coldstart.js | 41 +------------------ 3 files changed, 41 insertions(+), 72 deletions(-) diff --git a/test-infra/runner/framework/syncthing-control.js b/test-infra/runner/framework/syncthing-control.js index 3325e69c2f..8a8be69ca3 100644 --- a/test-infra/runner/framework/syncthing-control.js +++ b/test-infra/runner/framework/syncthing-control.js @@ -356,3 +356,42 @@ export async function resetSyncState() { export async function setFolderPatchDelay({ ip = '*', ms = 0 }) { return post('/folder-patch-delay', { ip, ms }); } + +// A folder in a true cold-start shape on every holder: an empty global of its own +// and no connected peer holding the data. Set before install, so the first election +// sees it rather than the stub's default phantom-synced peer. Honest only for a bare +// g:/r: component: an f:/m:/ml: mount leaves FluxOS scaffolding on the volume that a +// real daemon counts and the stub cannot see, so those shapes run on syncthing: 'binary'. +export async function pinColdStart(holders, folder) { + const subnet = getSubnetConfig(); + await Promise.all(holders.map((i) => Promise.all([ + setSyncState({ + ip: subnet.nodeIp(i + 1), folder, state: 'idle', globalBytes: 0, inSyncBytes: 0, receiveOnlyChangedFiles: 0, + }), + setNoPeerData({ ip: subnet.nodeIp(i + 1), folder }), + ]))); +} + +// The owner's data on one holder, declared as ENTRIES rather than a count, because +// localHoldings totals the entry list and excludes the scaffolding in it by type and +// by name. Written after pinColdStart, which it overwrites for this one node. +export async function pinHolding(index, folder, bytes) { + const subnet = getSubnetConfig(); + const modified = new Date().toISOString(); + await setSyncState({ + ip: subnet.nodeIp(index + 1), + folder, + state: 'idle', + globalBytes: 0, + inSyncBytes: 0, + localChanged: [ + { + name: 'appdata', type: 'FILE_INFO_TYPE_DIRECTORY', size: 128, deleted: false, modified, + }, + { + name: 'appdata/seed-data', type: 'FILE_INFO_TYPE_FILE', size: bytes, deleted: false, modified, + }, + ], + }); + await setNoPeerData({ ip: subnet.nodeIp(index + 1), folder }); +} diff --git a/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js b/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js index 99dc139e7d..a13e34fe5b 100644 --- a/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js +++ b/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js @@ -5,7 +5,7 @@ import { getAppContainerStatus, blockPeerAccess, unblockPeerAccess, execInContainer, } from '../framework/container.js'; import { - setSyncState, setNoPeerData, resetSyncState, getFolderConfig, + resetSyncState, getFolderConfig, pinColdStart, pinHolding, } from '../framework/syncthing-control.js'; import { resetFdm, electMaster } from '../framework/fdm-control.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; @@ -43,37 +43,6 @@ async function isUp(client, appName) { return !!(status && status.status.startsWith('Up')); } -// A cold start on every holder: no index of its own and no connected peer holding -// the data, set before install so the first election sees it. -async function pinColdStart(holders, folder) { - await Promise.all(holders.map((i) => Promise.all([ - setSyncState({ - ip: subnet.nodeIp(i + 1), folder, state: 'idle', globalBytes: 0, inSyncBytes: 0, receiveOnlyChangedFiles: 0, - }), - setNoPeerData({ ip: subnet.nodeIp(i + 1), folder }), - ]))); -} - -// The owner's data on one holder, declared as entries, the way localHoldings reads it. -async function pinHolding(index, folder, bytes) { - const modified = new Date().toISOString(); - await setSyncState({ - ip: subnet.nodeIp(index + 1), - folder, - state: 'idle', - globalBytes: 0, - inSyncBytes: 0, - localChanged: [ - { - name: 'appdata', type: 'FILE_INFO_TYPE_DIRECTORY', size: 128, deleted: false, modified, - }, - { - name: 'appdata/seed-data', type: 'FILE_INFO_TYPE_FILE', size: bytes, deleted: false, modified, - }, - ], - }); - await setNoPeerData({ ip: subnet.nodeIp(index + 1), folder }); -} describe('two holders that can reach each other never both seed', function () { let env; diff --git a/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js b/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js index 11f8d66c6e..3a3ce32b87 100644 --- a/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js +++ b/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js @@ -5,7 +5,7 @@ import { getAppContainerStatus, blockPeerAccess, unblockPeerAccess, execInContainer, } from '../framework/container.js'; import { - setSyncState, setNoPeerData, resetSyncState, + resetSyncState, pinColdStart, pinHolding, } from '../framework/syncthing-control.js'; import { electMaster, resetFdm } from '../framework/fdm-control.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; @@ -44,45 +44,6 @@ async function countUp(env, indices, appName) { return ups.filter(Boolean).length; } -// pin a folder to a true cold-start shape on every holder: empty global of its own -// and no connected peer holding the data (set BEFORE install so the first election -// evaluation sees it, not the stub's default phantom-synced peer) -async function pinColdStart(holders, folder) { - await Promise.all(holders.map((i) => Promise.all([ - setSyncState({ - ip: subnet.nodeIp(i + 1), folder, state: 'idle', globalBytes: 0, inSyncBytes: 0, - // Honest only for a bare g:/r: app. A component declaring an f:/m:/ml: mount has - // FluxOS scaffolding on its volume that a real daemon counts here, and the stub - // cannot see a disk - so those shapes belong on syncthing: 'binary', not here. - receiveOnlyChangedFiles: 0, - }), - setNoPeerData({ ip: subnet.nodeIp(i + 1), folder }), - ]))); -} - -// One holder carries the owner's data, declared the way seedSyncScopedData declares it: -// the ENTRIES, not a count, because localHoldings totals the entry list and excludes the -// scaffolding in it by type and by name. Written after pinColdStart, which it overwrites -// for this one node. -async function pinHolding(index, folder, bytes) { - const modified = new Date().toISOString(); - await setSyncState({ - ip: subnet.nodeIp(index + 1), - folder, - state: 'idle', - globalBytes: 0, - inSyncBytes: 0, - localChanged: [ - { - name: 'appdata', type: 'FILE_INFO_TYPE_DIRECTORY', size: 128, deleted: false, modified, - }, - { - name: 'appdata/seed-data', type: 'FILE_INFO_TYPE_FILE', size: bytes, deleted: false, modified, - }, - ], - }); - await setNoPeerData({ ip: subnet.nodeIp(index + 1), folder }); -} describe('reconciler cold start - fresh multi-node placement, no seeded source', function () { let env; From 2db3b19e20c1de52bf224dbbd26e7127f2234997 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:30:03 +0100 Subject: [PATCH 028/369] test(harness): 2250 a cold start seeds every g: component from the node holding any of its data A two-writer app cold-starts with the only data anywhere in one holder's copy of its second component's folder, that holder not the lowest address: it seeds and runs both components, the empty first folder included, and the lowest address seeds neither. Co-Authored-By: Claude Opus 5.5 --- ...t-from-the-node-holding-any-of-its-data.js | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js diff --git a/test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js b/test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js new file mode 100644 index 0000000000..b69bed8cd2 --- /dev/null +++ b/test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js @@ -0,0 +1,96 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { + resetSyncState, getFolderConfig, pinColdStart, pinHolding, +} from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { syncthingSeedIndex } from '../framework/g-app-placement.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A cold start of an app with two single-writer (g:) components, where the only +// data anywhere is one holder's copy of the SECOND component's folder. +// +// A holder ranks the claims on the whole app - every g: folder's holdings summed - +// so the node holding the owner's data seeds every g: component, the empty first +// folder included, and the lowest address seeds none. Ranked folder by folder, the +// empty first folder would fall to the address order and the app would be seeded on +// two nodes, one component each. +// +// The syncthing stub declares each folder's holdings, as suite 2214 does. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const SETTLE_PASSES = 3; +const HOLDING_BYTES = 5821604997; + + +describe('a cold start seeds every g: component from the node holding any of its data', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egcold${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const HOLDERS = [0, 1]; + const LOWEST = syncthingSeedIndex(HOLDERS); + const HOLDER = HOLDERS.find((i) => i !== LOWEST); + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const seeded = (i, name) => client(i).getDecisionCount('syncthing:seedDecision', folderOf(name), 'seeded'); + const folderPasses = (i, name) => client(i).getDecisionCount('syncthing:folderPass', folderOf(name), 'evaluated'); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + + await pushTestApp(appName, 'v1', 'gcold', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await Promise.all(COMPONENTS.map((name) => pinColdStart(HOLDERS, folderOf(name)))); + await pinHolding(HOLDER, folderOf('web'), HOLDING_BYTES); + await installOnNodes(env, app, HOLDERS); + }); + + after(async function () { + this.timeout(60000); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + it('seeds and runs every g: component on the holder of the second one\'s data, and nothing on the lowest address', async function () { + this.timeout(600000); + expect(subnet.nodeIp(LOWEST + 1) < subnet.nodeIp(HOLDER + 1), 'fixture: the holder of the data is not the lowest address').to.equal(true); + + await waitHolding(async () => (await oneHolderAtMost())[HOLDER].length === COMPONENTS.length, { + timeout: 300000, interval: 2000, label: 'the holder of the data runs every g: component', + }); + const from = await Promise.all(HOLDERS.flatMap((i) => COMPONENTS.map((name) => folderPasses(i, name)))); + await waitHolding(async () => { + expect(await oneHolderAtMost(), 'what each holder runs').to.deep.equal({ [HOLDER]: COMPONENTS, [LOWEST]: [] }); + const passes = await Promise.all(HOLDERS.flatMap((i) => COMPONENTS.map((name) => folderPasses(i, name)))); + return passes.every((n, k) => n >= from[k] + SETTLE_PASSES); + }, { timeout: 180000, interval: 1000, label: `${SETTLE_PASSES} folder passes over every g: folder on every holder` }); + + for (const name of COMPONENTS) { + // eslint-disable-next-line no-await-in-loop + expect(await seeded(HOLDER, name), `the holder seeded ${name}`).to.be.at.least(1); + // eslint-disable-next-line no-await-in-loop + expect(await seeded(LOWEST, name), `the lowest address seeded ${name}`).to.equal(0); + // eslint-disable-next-line no-await-in-loop + expect((await getFolderConfig(subnet.nodeIp(LOWEST + 1), folderOf(name)))?.type, `the lowest address's ${name} folder`).to.equal('receiveonly'); + } + }); +}); From 7dbbf1f30cafc3f27eae869ff6af20161e2734cf Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:32:18 +0100 Subject: [PATCH 029/369] test(harness): 2249 a surplus trim keeps the copy running any g: component The newest copy of a two-writer app is its primary, running the second g: component with the first stopped there by its owner, and a stub peer makes the copies one too many: the newest declines to stand aside, and the next copy is trimmed instead. Co-Authored-By: Claude Opus 5.5 --- ...-keeps-the-copy-running-any-g-component.js | 195 ++++++++++++++++++ 1 file changed, 195 insertions(+) create mode 100644 test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js diff --git a/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js b/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js new file mode 100644 index 0000000000..1a2d2f296f --- /dev/null +++ b/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js @@ -0,0 +1,195 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, electionOrder } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding, waitForGiveUpConsidered } from '../framework/wait.js'; +import { advanceBlock, driveUntil, startTicker, stopTicker } from '../framework/daemon-control.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, +} from '../framework/syncthing-real.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Trimming a surplus copy of an app with two single-writer (g:) components, when +// the copy the surplus rule picks - the newest - is the app's primary, running its +// second g: component with the first stopped there by its owner. Real syncthing +// daemons. +// +// The newest copy stands aside unless it runs the app's writer, and every g: +// component is the writer: the primary runs them together. So the newest copy +// stays while it runs ANY of them, and the next copy trims instead, on the +// newest's own word that it holds the app (suite 96). +// +// THE SHAPE: the first holder places and seeds the app, the second takes it over +// when the first dies, and the first comes back as a standby - so the primary is +// the newest copy. A stub peer then announces the app, backdated so it is the most +// senior holder: three holders against an instance count of two. + +const APP_UID = 1000; +const OLDER = 0; +const NEWEST = 1; +const RING = 2; +const STUB_INDEX = 3; +const HOLDERS = [OLDER, NEWEST]; + +describe('a surplus trim keeps the copy running any g: component', function () { + let env; + let held; + dumpLogsOnFailure(() => env); + + const appName = `e2egsurplus${Date.now()}`; + const COMPONENTS = ['db', 'web']; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const ownerAuth = async (i) => (await authenticate(client(i).url, appOwnerKey())).zelidauth; + const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(i), folderOf(name)))?.type)); + // Read from a holder's own list, senior first. + const order = async (via) => { + const res = await client(via).getAppLocations(appName); + if (res?.status !== 'success' || !Array.isArray(res.data) || !res.data.length) { + throw new Error(`app locations unreadable: ${JSON.stringify(res)?.slice(0, 200)}`); + } + return electionOrder(res.data).map((location) => location.ip.split(':')[0]); + }; + // Whether a holder has the app installed, from an answer it gave: a read that + // fails is asked again, never taken as the app gone. + const installedOn = async (i) => { + let installed = null; + await waitFor(async () => { + const res = await client(i).getInstalledApps(); + if (res?.status !== 'success' || !Array.isArray(res.data)) throw new Error(`installed apps unreadable on node ${i}: ${JSON.stringify(res)?.slice(0, 200)}`); + installed = res.data.some((a) => a.name === appName); + return true; + }, { timeout: 60000, interval: 2000, label: `node ${i} answers what it has installed` }); + return installed; + }; + // What the decision reads, asked the way the product asks it: the newest copy's + // /apps/heldcomponents, from the next copy. + const newestAnswersHeld = async () => { + const res = await execInContainer(client(OLDER).container, `curl -sf -m 5 http://${client(NEWEST).ip}:16127/apps/heldcomponents`); + return res.exitCode === 0 && res.output.includes(folderOf('web')); + }; + + before(async function () { + this.timeout(1500000); + env = await createTestEnv({ + hookCtx: this, + nodes: STUB_INDEX + 1, + stubPeers: [STUB_INDEX], + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all([OLDER, NEWEST, RING].map((i) => waitFor(() => isDaemonUp(client(i)), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gsurplus', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [OLDER]); + await waitHolding(async () => (await oneHolderAtMost())[OLDER].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + await installOnNodes(env, app, [NEWEST]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(NEWEST), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the second holder has the first holder's ${name} data`, + }))); + + // The first holder dies whole, the second takes the app over, and the first + // comes back as its standby. + await crashFluxos(client(OLDER).container, { hold: true }); + await Promise.all(COMPONENTS.map((name) => execInContainer(client(OLDER).container, `docker kill ${folderOf(name)}`))); + await stopDaemon(client(OLDER)); + await waitHolding(async () => (await oneHolderAtMost())[NEWEST].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the second holder runs every g: component', + }); + await startDaemon(client(OLDER), { paused: true }); + await releaseFluxos(client(OLDER).container); + await waitHolding(async () => { + await oneHolderAtMost(); + return (await folderTypes(OLDER)).every((type) => type === 'receiveonly'); + }, { timeout: 300000, interval: 2000, label: 'the first holder is back as a standby of the whole app' }); + + // Its owner stops the first g: component alone on the primary. + const res = await client(NEWEST).getAuthed(`/apps/appstop/db_${appName}`, await ownerAuth(NEWEST)); + expect(res?.status, `owner stop of db on the primary: ${JSON.stringify(res)}`).to.equal('success'); + await waitHolding(async () => JSON.stringify((await oneHolderAtMost())[NEWEST]) === JSON.stringify(['web']), { + timeout: 180000, interval: 2000, label: 'the primary runs web alone', + }); + expect(await order(OLDER), 'fixture: the primary is the newest copy').to.deep.equal([client(OLDER).ip, client(NEWEST).ip]); + await waitFor(newestAnswersHeld, { + timeout: 180000, interval: 3000, label: 'the newest copy answers heldcomponents with web, asked from the next copy', + }); + + // The surplus has to survive until the test drives the trim pass, which runs + // on block heights: every node is brought level with a frozen tip first, so + // none processes its way onto one while the fixture is assembling. + await stopTicker(); + const real = [OLDER, NEWEST, RING]; + const seen = real.map((i) => client(i).getLastEventId()); + const { currentHeight } = await advanceBlock(); + await Promise.all(real.map((i, k) => client(i).waitForEvent( + 'block:processed', (d) => d.height >= currentHeight, 120000, { afterId: seen[k] }, + ))); + + const stub = env.stubPeerClients.get(STUB_INDEX); + await waitFor(async () => (await stub.connectedNodes()) > 0, { + timeout: 120000, interval: 2000, label: 'the fleet has connected to the stub peer', + }); + held = stub.holdApp(appName, { hash: app.hash, runningSince: Date.now() - 24 * 60 * 60 * 1000 }); + await held.started; + await Promise.all(HOLDERS.map((i) => waitFor(async () => (await order(i)).length === 3, { + timeout: 120000, interval: 2000, label: `holder ${i} counts the stub as a third holder`, + }))); + }); + + after(async function () { + this.timeout(120000); + held?.stop(); + await startTicker().catch(() => {}); + await env?.teardown(); + }); + + it('keeps the newest copy, which runs the second g: component with the first stopped, and trims the next copy instead', async function () { + this.timeout(900000); + let kept = null; + const keeping = waitForGiveUpConsidered( + client(NEWEST), appName, + (d) => d.giveUp === false && d.reason === 'SURPLUS' && d.code === 'NEWEST_HOLDS_WRITER', + 600000, + ).then((v) => { kept = v; return v; }); + keeping.catch(() => {}); + + // The pass runs every removeFluxAppsPeriod * speedMultiplier blocks and this + // drive spends about one period, so there is one attempt: a block is driven + // only while the newest copy is answering what the next copy will ask it. + await driveUntil(client(OLDER), async () => { + if (!await installedOn(NEWEST)) return true; + if (!await installedOn(OLDER)) return kept !== null; + await waitFor(newestAnswersHeld, { + timeout: 120000, interval: 3000, label: 'the newest copy still answers heldcomponents before another block is driven', + }); + return false; + }, { timeoutMs: 600000, label: 'a copy is trimmed and the newest has declined to stand aside' }); + await startTicker(); + + expect(await installedOn(NEWEST), 'the newest copy, which runs web, is still installed').to.equal(true); + expect(kept?.code, 'the newest copy declined to stand aside').to.equal('NEWEST_HOLDS_WRITER'); + expect(await installedOn(OLDER), 'the next copy is still installed').to.equal(false); + expect(await oneHolderAtMost(), 'what each holder runs').to.deep.equal({ [OLDER]: [], [NEWEST]: ['web'] }); + }); +}); From 25af17a4ab1259a945b32445dad1285b83cc7544 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 17:53:03 +0100 Subject: [PATCH 030/369] test(harness): 2249 reads the newest copy's decision from the event's data waitForEvent resolves with the stream entry, whose data carries the decision; the wait is anchored at the test's start. Co-Authored-By: Claude Opus 5.5 --- ...-a-surplus-trim-keeps-the-copy-running-any-g-component.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js b/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js index 1a2d2f296f..d7f4737205 100644 --- a/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js +++ b/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js @@ -169,9 +169,10 @@ describe('a surplus trim keeps the copy running any g: component', function () { let kept = null; const keeping = waitForGiveUpConsidered( client(NEWEST), appName, - (d) => d.giveUp === false && d.reason === 'SURPLUS' && d.code === 'NEWEST_HOLDS_WRITER', + (d) => d.giveUp === false && d.reason === 'SURPLUS', 600000, - ).then((v) => { kept = v; return v; }); + { afterId: client(NEWEST).getLastEventId() }, + ).then((entry) => { kept = entry.data; return entry; }); keeping.catch(() => {}); // The pass runs every removeFluxAppsPeriod * speedMultiplier blocks and this From e1a7b4ff03f94b13b7d8bef63ff2438e1b296055 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 18:48:58 +0100 Subject: [PATCH 031/369] fix(update): an update restarts an app through the reconciler An update that rebuilds an app on a node restarts it there. That restart is now a restart request per component, written inside the reconciler's slot for it: the reconciler bounces each component it keeps running on the node and starts none it keeps stopped. A standby's g: component stays down and the election decides where it runs; a component its owner stopped stays stopped. docker restart starts a stopped container, so restarting every container started a standby's g: components beside the primary's after any update the node applied in place, and after a version upgrade. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 71 +++++-------------- tests/unit/advancedWorkflows.test.js | 43 +++++++++-- tests/unit/appReconciler.test.js | 25 +++++++ 3 files changed, 80 insertions(+), 59 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index e09033b281..6f3de20123 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -2540,64 +2540,29 @@ async function appDockerStop(appname, onWait) { } /** - * Helper function to restart app docker containers - * Ensures mount paths exist before restarting (important after Syncthing cleanup) - * @param {string} appname - App name + * Restarts an app once an update has rebuilt it, through the reconciler: each + * component gets a restart request, and the reconciler bounces the ones it keeps + * running on this node and starts none it keeps stopped. A g: component on a + * standby stays down - the election decides where it runs - and so does one its + * owner stopped. A stopped component the reconciler does start counts as + * restarted, so it is not bounced again once it runs. + * @param {string} appname App name, or `_` for one component. * @returns {Promise} */ async function appDockerRestart(appname) { - // eslint-disable-next-line global-require try { - // eslint-disable-next-line global-require - const { startAppMonitoring } = require('../appManagement/appInspector'); - // eslint-disable-next-line global-require - const registryManager = require('../appDatabase/registryManager'); - const mainAppName = appname.split('_')[1] || appname; - const isComponent = appname.includes('_'); - if (isComponent) { - // For component apps, fetch full specifications to ensure mount paths exist - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - // Find the specific component - const componentName = appname.split('_')[0]; - const componentSpec = appSpecs.compose.find((comp) => comp.name === componentName); - if (componentSpec && componentSpec.containerData) { - // Ensure mount paths exist before restarting (handles Syncthing cleanup) - // eslint-disable-next-line no-use-before-define - await volumeService.ensureMountPathsExist(componentSpec, mainAppName, true, appSpecs); - } - await dockerService.appDockerRestart(appname); - startAppMonitoring(appname); - } else { - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - if (appSpecs.version <= 3) { - // Ensure mount paths exist before restarting (handles Syncthing cleanup) - if (appSpecs.containerData) { - // eslint-disable-next-line no-use-before-define - await volumeService.ensureMountPathsExist(appSpecs, mainAppName, false, null); - } - await dockerService.appDockerRestart(appname); - startAppMonitoring(appname); - } else { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - // Ensure mount paths exist before restarting (handles Syncthing cleanup) - // eslint-disable-next-line no-await-in-loop - if (appComponent.containerData) { - // eslint-disable-next-line no-await-in-loop, no-use-before-define - await volumeService.ensureMountPathsExist(appComponent, appSpecs.name, true, appSpecs); - } - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerRestart(`${appComponent.name}_${appSpecs.name}`); - startAppMonitoring(`${appComponent.name}_${appSpecs.name}`); - } - } + const appSpecs = await registryManagerModule.getApplicationSpecifications(mainAppName); + if (!appSpecs) { + throw new Error('Application not found'); + } + const identifiers = appname.includes('_') || appSpecs.version <= 3 + ? [appname] + : appSpecs.compose.map((appComponent) => `${appComponent.name}_${appSpecs.name}`); + // eslint-disable-next-line no-restricted-syntax + for (const identifier of identifiers) { + // eslint-disable-next-line no-await-in-loop + await appReconciler.applyIntent(identifier, () => appsRuntimeState.requestRestart(identifier)); } } catch (error) { log.error(error); diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index aa2babcc11..9d83e8faa1 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -7408,10 +7408,12 @@ describe('advancedWorkflows tests', () => { // The node records the new specification before it builds the replacement, // so the row cannot say the reinstall is done. The announcement does, once - // every replaced component is back and the app restarted. + // every replaced component is back and the app's restart requested. describe('announcing the replaced components', () => { let publish; - let appDockerRestart; + let dockerRestart; + let requestRestart; + let applyIntent; let installApplicationSoft; beforeEach(() => { @@ -7426,13 +7428,18 @@ describe('advancedWorkflows tests', () => { sinon.stub(registryManager, 'getApplicationSpecifications').resolves(newSpec); sinon.stub(volumeService, 'ensureMountPathsExist').resolves(); sinon.stub(appInspector, 'startAppMonitoring'); - appDockerRestart = sinon.stub(dockerServiceModule, 'appDockerRestart').resolves(); + dockerRestart = sinon.stub(dockerServiceModule, 'appDockerRestart').resolves(); + requestRestart = sinon.stub(appsRuntimeState, 'requestRestart').resolves(); + applyIntent = sinon.stub(appReconciler, 'applyIntent').callsFake(async (identifier, mutate) => { + await mutate(); + return true; + }); publish = sinon.spy(fluxEventBus, 'publish'); }); const redeployedEvents = () => publish.getCalls().filter((c) => c.args[0] === 'app:componentRedeployed'); - it('announces each replaced component after the app is back and restarted', async () => { + it('announces each replaced component after the app is back and its restart requested', async () => { await advancedWorkflows.reinstallOldApplications(); const events = redeployedEvents(); @@ -7440,8 +7447,32 @@ describe('advancedWorkflows tests', () => { expect(events[0].args[1]).to.deep.equal({ name: 'myapp', component: 'web', identifier: 'web_myapp', hard: false, }); - sinon.assert.calledWith(appDockerRestart, 'web_myapp'); - expect(appDockerRestart.lastCall.calledBefore(events[0]), 'announced after the restart').to.equal(true); + sinon.assert.calledWith(requestRestart, 'web_myapp'); + expect(requestRestart.lastCall.calledBefore(events[0]), 'announced after the restart is requested').to.equal(true); + }); + + // The reconciler decides what runs on this node: it bounces a component it + // keeps running and starts none it keeps stopped, so a standby's g: + // component is not started beside its primary's. + it('restarts the app through the reconciler, never by docker', async () => { + await advancedWorkflows.reinstallOldApplications(); + + expect(requestRestart.called, 'the fixture reached the restart').to.equal(true); + sinon.assert.calledWith(applyIntent, 'web_myapp', sinon.match.func); + expect(applyIntent.firstCall.calledBefore(requestRestart.firstCall), 'the request is written inside the reconciler\'s slot').to.equal(true); + expect(dockerRestart.called, 'docker restarted a container').to.equal(false); + }); + + it('requests a restart of every component of the app, its g: components included', async () => { + registryManager.getApplicationSpecifications.resolves({ + ...newSpec, + compose: [...newSpec.compose, { ...newSpec.compose[0], name: 'db', containerData: 'g:/data' }], + }); + + await advancedWorkflows.reinstallOldApplications(); + + expect(requestRestart.args.map(([identifier]) => identifier)).to.deep.equal(['web_myapp', 'db_myapp']); + expect(applyIntent.args.map(([identifier]) => identifier)).to.deep.equal(['web_myapp', 'db_myapp']); }); it('announces nothing when a component does not come back', async () => { diff --git a/tests/unit/appReconciler.test.js b/tests/unit/appReconciler.test.js index c2d7f6da39..7927fb1cdf 100644 --- a/tests/unit/appReconciler.test.js +++ b/tests/unit/appReconciler.test.js @@ -822,6 +822,31 @@ describe('appReconciler tests', () => { expect(stubs.dockerService.appDockerStart.called).to.be.false; }); + // An update requests a restart of every component on every node that holds the + // app. On a standby that request must not start the g: component: the election + // decides where it runs. It stays pending, and a start the election later asks + // for satisfies it. + it('does not start a stopped g: component for a restart request while no controller has elected it', async () => { + localSpec = { name: 'App', version: 4, compose: [{ name: 'db', containerData: 'g:/data' }] }; + stubs.appsRuntimeState.getState.resolves({ restartGeneration: 2, actuatedRestartGeneration: 1 }); + await appReconciler.reconcile('db_App'); // controllerDesired unset, inspect default: stopped + expect(stubs.dockerService.appDockerStart.called, 'started').to.be.false; + expect(stubs.dockerService.appDockerRestart.called, 'restarted').to.be.false; + expect(stubs.appsRuntimeState.recordRestartGeneration.called, 'the request recorded as done').to.be.false; + }); + + it('does not start a stopped g: component for a restart request once a controller stood it down', async () => { + localSpec = { name: 'App', version: 4, compose: [{ name: 'db', containerData: 'g:/data' }] }; + stubs.globalState.bootContainerStateSettled = false; + appReconciler.setControllerDesired('db_App', 'stopped', 'masterSlave standby'); + stubs.globalState.bootContainerStateSettled = true; + stubs.appsRuntimeState.getState.resolves({ restartGeneration: 2, actuatedRestartGeneration: 1 }); + await appReconciler.reconcile('db_App'); // inspect default: stopped + expect(stubs.dockerService.appDockerStart.called, 'started').to.be.false; + expect(stubs.dockerService.appDockerRestart.called, 'restarted').to.be.false; + expect(stubs.appsRuntimeState.recordRestartGeneration.called, 'the request recorded as done').to.be.false; + }); + it('starts a g: component once a controller sets it running', async () => { localSpec = { name: 'App', version: 4, compose: [{ name: 'db', containerData: 'g:/data' }] }; // set desired without triggering the workqueue (boot gate closed -> enqueue held) From 3a1e4c3d4705b71bb2f5316ed2c2de576c1db7f6 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 18:49:52 +0100 Subject: [PATCH 032/369] test(harness): 2251 an update applied in place leaves a standby's g: component stopped An update that changes only the plain sibling of an app's g: component is applied in place on both holders - the sibling redeployed, then the app restarted: the standby's g: component stays stopped through it and after it, the primary goes on running it, and the sibling runs on both. Co-Authored-By: Claude Opus 5.5 --- ...e-leaves-a-standbys-g-component-stopped.js | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js diff --git a/test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js b/test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js new file mode 100644 index 0000000000..9eb87ac8ed --- /dev/null +++ b/test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js @@ -0,0 +1,126 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A specification update that changes only the plain sibling of an app's single +// single-writer (g:) component, on real syncthing daemons. +// +// Each holder applies it in place: it redeploys the sibling and then restarts the +// app. The restart bounces what this node runs and starts nothing it keeps +// stopped, so the standby's g: component stays down through the update and after +// it, and the primary goes on running it. Started, the app would run on both +// nodes. The plain sibling runs on every holder throughout. +// +// DRIVEN BY BLOCKS: the reinstall pass fires on +// `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). + +const APP_UID = 1000; + +describe('an update applied in place leaves a standby\'s g: component stopped', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2einplace${Date.now()}`; + const PRIMARY = 0; + const STANDBY = 1; + const HOLDERS = [PRIMARY, STANDBY]; + const web = (content) => ({ + ...gComponent(appName, 'web'), + containerData: '/appdata', + environmentParameters: ['WRITE_FILE=/appdata/web.txt', `WRITE_CONTENT=${content}`], + }); + let app; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: ['db'], holders: HOLDERS, + }); + const plain = watchGComponents(() => env, { appName, components: ['web'], holders: HOLDERS }); + const installedHash = async (i) => { + const res = await client(i).getInstalledApps(); + return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; + }; + const redeployed = (i, from) => client(i).getEventBuffer().some((e) => e.id > from + && e.event === 'app:componentRedeployed' && e.data?.name === appName && e.data?.component === 'web'); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'inplace', { user: APP_UID }); + app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: [gComponent(appName, 'db'), web('first')], + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + await waitFor(async () => (await plain.runningOn(STANDBY)).includes('web'), { + timeout: 180000, interval: 3000, label: 'the standby runs the plain sibling', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('keeps the standby\'s g: component stopped through an update that redeploys its sibling in place', async function () { + // A backstop: the block drive bounds this test (suite 85). + this.timeout(1800000); + const updated = await buildSeedableUpdate(app, (spec) => { + spec.compose[1].environmentParameters = web('second').environmentParameters; + }); + const from = Object.fromEntries(HOLDERS.map((i) => [i, client(i).getLastEventId()])); + await seedSpecUpdate(env, updated, HOLDERS); + await stopTicker(); + + let breach = null; + await driveUntil(client(PRIMARY), async () => { + try { + await oneHolderAtMost(); + const hashes = await Promise.all(HOLDERS.map(installedHash)); + return hashes.every((hash) => hash === updated.hash) && HOLDERS.every((i) => redeployed(i, from[i])); + } catch (error) { + if (error?.name !== 'AssertionError') throw error; + breach = error; + return true; + } + }, { blocks: 216, label: 'both holders redeployed the sibling in place at the new specification' }); + if (breach) throw breach; + + // The restart each holder asked for, carried out. + await holdsFor(60000, { [PRIMARY]: ['db'], [STANDBY]: [] }); + expect((await getFolderConfig(client(STANDBY), folderOf('db')))?.type, 'the standby\'s db folder').to.equal('receiveonly'); + expect((await getFolderConfig(client(PRIMARY), folderOf('db')))?.type, 'the primary\'s db folder').to.equal('sendreceive'); + for (const i of HOLDERS) { + // eslint-disable-next-line no-await-in-loop + expect(await plain.runningOn(i), `node ${i} runs the plain sibling`).to.deep.equal(['web']); + } + }); +}); From 16ce6de8d552bdbd046f2ae22e55b35ca85e6733 Mon Sep 17 00:00:00 2001 From: Valter Silva Date: Sat, 17 Oct 2026 20:14:15 +0100 Subject: [PATCH 033/369] feat(apps): never throttle the cpu of a games-hub game server checkApplicationsCpuUSage lowers any app with a spec above 2 cpus to 80% once it is above 92% on 80% of its window, and measures the next window against that reduced allocation, so a saturated app never earns it back. For a game server that is the wrong trade: a game's tick is a latency budget, and the cut turns into lag and rubber-banding for its players. An Enshrouded server on a 3-cpu spec was found held at 2.4 cpus with its tick overrunning by up to 2x. A new signed policy document, cputhrottleexempt, lists the app name prefixes of the games hub's game servers (the lowercased marketplace names). The hub names every server it registers , so an app is exempt only when the name splits into a listed prefix and a 13-digit timestamp between 2023 and a day past now. Exempt containers are held at their full specified cpu, and one already lowered is restored at once, without waiting for a full window. Exemption never grants more than the spec, so a name that qualifies without being a game gains nothing it was not already owed. With no document, or a malformed one, nothing is exempt and the throttle behaves exactly as before, so the document can be published ahead of this. Co-Authored-By: Claude Opus 5.5 --- .../services/appManagement/appInspector.js | 42 ++++++ .../services/utils/cpuThrottleExemption.js | 95 ++++++++++++++ tests/unit/appInspector.test.js | 95 ++++++++++++++ tests/unit/cpuThrottleExemption.test.js | 120 ++++++++++++++++++ 4 files changed, 352 insertions(+) create mode 100644 ZelBack/src/services/utils/cpuThrottleExemption.js create mode 100644 tests/unit/cpuThrottleExemption.test.js diff --git a/ZelBack/src/services/appManagement/appInspector.js b/ZelBack/src/services/appManagement/appInspector.js index 4de7b493d6..cdbadf3a07 100644 --- a/ZelBack/src/services/appManagement/appInspector.js +++ b/ZelBack/src/services/appManagement/appInspector.js @@ -7,6 +7,7 @@ const logCursor = require('../utils/logCursor'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const globalState = require('../utils/globalState'); const cpuBurstHelper = require('../utils/cpuBurstHelper'); +const cpuThrottleExemption = require('../utils/cpuThrottleExemption'); const log = require('../../lib/log'); const { getContainerStorage } = require('../utils/appUtilities'); const { Privilege, authOf } = require('../utils/privileges'); @@ -908,6 +909,33 @@ function sampleCpuLoad(sample, specifiedCpu) { return ((cpuUsage / systemCpuUsage) * sample.onlineCpus * 100) / specifiedCpu || 0; } +/** + * Hold a throttle-exempt container at its full specified cpu instead of deciding on it. + * + * Restoring is not optional. A game server lowered before it became exempt (or before this + * node read the policy) would otherwise stay at 80% forever: the decision below measures load + * against the reduced allocation, so a saturated server never looks idle enough to be raised. + * @param {string} containerName - The app or component container name + * @param {number} specifiedCpu - The cpu the app or component specified + * @param {object} inspect - The container's docker inspect + * @param {object} monitored - The monitoring entry, which may have gone mid-pass + * @param {number} decisionAt - The watermark to stamp + * @returns {Promise} + */ +async function holdExemptAtSpecifiedCpu(containerName, specifiedCpu, inspect, monitored, decisionAt) { + const nanoCpus = inspect.HostConfig?.NanoCpus; + const specifiedNanoCpus = Math.round(specifiedCpu * 1e9); + if (nanoCpus && nanoCpus < specifiedNanoCpus) { + await dockerService.appDockerUpdateCpu(containerName, specifiedNanoCpus); + log.info(`checkApplicationsCpuUSage ${containerName} throttle-exempt, restoring cpu 100.`); + } + // Guarded for the same reason as the watermark writes in the caller. + if (monitored) { + // eslint-disable-next-line no-param-reassign + monitored.lastCpuDecisionAt = decisionAt; + } +} + /** * Check if applications are throttling CPU and adjust CPU limits * @param {object} appsMonitored - Applications monitoring data @@ -951,6 +979,13 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { // eslint-disable-next-line no-continue continue; } + // Game servers deployed by the games hub are never throttled. See cpuThrottleExemption. + if (inspect && cpuThrottleExemption.isCpuThrottleExempt(app.name)) { + // eslint-disable-next-line no-await-in-loop + await holdExemptAtSpecifiedCpu(app.name, app.cpu, inspect, appsMonitored[app.name], decisionAt); + // eslint-disable-next-line no-continue + continue; + } if (inspect && stats && stats.length > 4) { const nanoCpus = inspect.HostConfig.NanoCpus; let cpuThrottlingRuns = 0; @@ -1031,6 +1066,13 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { // eslint-disable-next-line no-continue continue; } + // Exemption is by APP name: every component of a hub game server is held at spec. + if (inspect && cpuThrottleExemption.isCpuThrottleExempt(app.name)) { + // eslint-disable-next-line no-await-in-loop + await holdExemptAtSpecifiedCpu(compName, appComponent.cpu, inspect, appsMonitored[compName], decisionAt); + // eslint-disable-next-line no-continue + continue; + } if (inspect && stats && stats.length > 4) { const nanoCpus = inspect.HostConfig.NanoCpus; let cpuThrottlingRuns = 0; diff --git a/ZelBack/src/services/utils/cpuThrottleExemption.js b/ZelBack/src/services/utils/cpuThrottleExemption.js new file mode 100644 index 0000000000..98094264f7 --- /dev/null +++ b/ZelBack/src/services/utils/cpuThrottleExemption.js @@ -0,0 +1,95 @@ +const log = require('../../lib/log'); +const policyStore = require('../policyStore'); + +// Which apps the CPU throttle leaves at their full specified allocation, read from the signed +// policy bundle. The document lists app name prefixes: the lowercased marketplace names of the +// game servers the games hub deploys. The hub names every server it registers +// ``, so a prefix alone is not enough to qualify -- the rest of +// the name must be exactly that millisecond timestamp, and a plausible one. +// +// Game servers are exempt because a game's tick is a latency budget, not a throughput job. +// Taking 20% of a saturated server's cpu does not slow it down evenly, it makes every tick that +// overruns the budget arrive late, which players feel as lag and rubber-banding. The throttle +// then never lets go: it measures load against the reduced allocation, so a server that was +// saturated stays saturated, and stays at 80%. +// +// Exemption only ever holds an app at what its owner specified and paid for. It never grants +// more, so a name that qualifies without being a game gains nothing it was not already owed. +// +// Unknown policy answers "not exempt": the throttle is the default, and a node that has not read +// policy keeps doing what it did before this document existed. + +const DOCUMENT = 'cputhrottleexempt'; + +// Date.now() in milliseconds is 13 digits from 2001 until the year 2286. +const TIMESTAMP_DIGITS = 13; +// The earliest game server the hub, or the dedicated sites before it, could have named. +const EARLIEST_TIMESTAMP_MS = Date.UTC(2023, 0, 1); +// A name is minted in the customer's browser before registration, so its timestamp can sit a +// little ahead of this node's clock. A day absorbs any skew; beyond that it is not a timestamp. +const FUTURE_TOLERANCE_MS = 24 * 60 * 60 * 1000; + +const PREFIX_PATTERN = /^[a-z0-9]+$/; + +// Memoized prefix set, keyed by the document it was built from. The store hands back the same +// object until it adopts a new bundle, so reference identity is a sound signal. +let prefixCache = null; +let prefixCacheKey = null; + +/** + * A valid document is an array of lowercase alphanumeric prefixes. + * + * Still checked even though the bundle is signed: a signature says who published a document, + * not that it is the shape this code expects. + */ +function isValidPrefixList(value) { + return Array.isArray(value) + && value.every((prefix) => typeof prefix === 'string' && PREFIX_PATTERN.test(prefix)); +} + +/** + * The exempt prefixes as a Set, or null when the policy is unknown or malformed. + * @returns {Set|null} + */ +function getExemptPrefixes() { + const document = policyStore.getDocument(DOCUMENT); + if (document === null) return null; + if (prefixCacheKey === document) return prefixCache; + if (!isValidPrefixList(document)) { + log.error(`cpuThrottleExemption - ${DOCUMENT} in the signed bundle is not an array of lowercase alphanumeric prefixes`); + return null; + } + prefixCache = new Set(document); + prefixCacheKey = document; + return prefixCache; +} + +/** + * Whether an app is a hub game server the CPU throttle must leave alone. + * + * The timestamp is the last 13 characters, and the prefix is everything before them. Splitting + * from the end rather than matching the prefix first is what keeps a prefix that itself ends in + * a digit (eurotrucksimulator2) unambiguous, and keeps one prefix from claiming another's apps + * (rustserver does not cover rustserveroxide; each is listed on its own). + * @param {string} appName - The application name + * @param {number} [now] - The current time in ms, for tests + * @returns {boolean} + */ +function isCpuThrottleExempt(appName, now = Date.now()) { + if (typeof appName !== 'string' || appName.length <= TIMESTAMP_DIGITS) return false; + const prefixes = getExemptPrefixes(); + if (prefixes === null || prefixes.size === 0) return false; + + const name = appName.toLowerCase(); + const timestamp = name.slice(-TIMESTAMP_DIGITS); + if (!/^\d+$/.test(timestamp)) return false; + const ms = Number(timestamp); + if (ms < EARLIEST_TIMESTAMP_MS || ms > now + FUTURE_TOLERANCE_MS) return false; + + return prefixes.has(name.slice(0, -TIMESTAMP_DIGITS)); +} + +module.exports = { + getExemptPrefixes, + isCpuThrottleExempt, +}; diff --git a/tests/unit/appInspector.test.js b/tests/unit/appInspector.test.js index 1420160466..68b9a7250b 100644 --- a/tests/unit/appInspector.test.js +++ b/tests/unit/appInspector.test.js @@ -14,6 +14,7 @@ describe('appInspector tests', () => { let configStub; let globalStateStub; let cpuBurstHelperStub; + let cpuThrottleExemptionStub; let appUtilitiesStub; beforeEach(() => { @@ -38,6 +39,10 @@ describe('appInspector tests', () => { isBurstActive: sinon.stub().resolves(false), }; + cpuThrottleExemptionStub = { + isCpuThrottleExempt: sinon.stub().returns(false), + }; + // Hoisted so a test can drive the disk reading. A partial one is a floor // rather than a total, and what the store does with it is the difference // between a chart and a cliff. @@ -87,6 +92,7 @@ describe('appInspector tests', () => { }, '../utils/appUtilities': appUtilitiesStub, '../utils/cpuBurstHelper': cpuBurstHelperStub, + '../utils/cpuThrottleExemption': cpuThrottleExemptionStub, 'node-cmd': { run: (cmd, callback) => callback(null, 'data', 'stderr'), }, @@ -2359,6 +2365,95 @@ describe('appInspector tests', () => { expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly('db_myapp', 1.8e9)).to.be.true; }); + + // Hub game servers. A game's tick is a latency budget, so the 80% cut shows up as lag, + // and a saturated server measured against its reduced allocation never earns it back. + describe('throttle-exempt apps', () => { + const gameApp = { name: 'enshrouded1792177003381', version: 3, cpu: 3 }; + + beforeEach(() => { + cpuThrottleExemptionStub.isCpuThrottleExempt.callsFake((name) => name === gameApp.name); + }); + + it('never lowers a saturated exempt app', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 3e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.called).to.be.false; + }); + + it('restores an exempt app already lowered to 80%, even while it is saturated', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 2.4e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly(gameApp.name, 3e9)).to.be.true; + }); + + it('restores without waiting for a full window of samples', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 2.4e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: [] } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly(gameApp.name, 3e9)).to.be.true; + }); + + it('consumes the window, so the samples it declined to judge are never held against it', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 3e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]), lastCpuDecisionAt: 0 } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(globalStateStub.appsMonitored[gameApp.name].lastCpuDecisionAt).to.be.above(0); + }); + + it('still throttles an app that is not exempt in the same pass', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 2e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { + [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) }, + myapp: { statsStore: window([1, 1, 1, 1, 1]) }, + }; + const both = sinon.stub().resolves({ status: 'success', data: [{ ...gameApp, cpu: 2 }, simpleApp] }); + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, both); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly('myapp', 1.8e9)).to.be.true; + }); + + it('holds every component of an exempt composed app at its own spec', async () => { + const composed = { + name: 'palworld1792177003381', + version: 8, + compose: [{ name: 'game', cpu: 4 }, { name: 'sidecar', cpu: 2 }], + }; + cpuThrottleExemptionStub.isCpuThrottleExempt.callsFake((name) => name === composed.name); + dockerServiceStub.dockerContainerInspect.callsFake(async (name) => ({ + HostConfig: { NanoCpus: name.startsWith('game_') ? 3.2e9 : 2e9 }, + State: { Pid: 1234 }, + })); + globalStateStub.appsMonitored = { + game_palworld1792177003381: { statsStore: window([1, 1, 1, 1, 1]) }, + sidecar_palworld1792177003381: { statsStore: window([1, 1, 1, 1, 1]) }, + }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(composed)); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly('game_palworld1792177003381', 4e9)).to.be.true; + }); + + it('leaves the container alone when docker cannot inspect it', async () => { + dockerServiceStub.dockerContainerInspect.resolves(null); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.called).to.be.false; + }); + }); }); describe('exported functions', () => { diff --git a/tests/unit/cpuThrottleExemption.test.js b/tests/unit/cpuThrottleExemption.test.js new file mode 100644 index 0000000000..976b4a8c6d --- /dev/null +++ b/tests/unit/cpuThrottleExemption.test.js @@ -0,0 +1,120 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +const MODULE_PATH = '../../ZelBack/src/services/utils/cpuThrottleExemption'; + +const PREFIXES = ['enshrouded', 'palworld', 'rustserver', 'rustserveroxide', 'eurotrucksimulator2', '7daystodie']; + +// A fixed clock, so a timestamp "in the future" means the same thing on every run. +const NOW = Date.UTC(2026, 9, 17, 18, 0, 0); +const TS = String(Date.UTC(2026, 9, 16, 18, 56, 43, 381)); + +function load(document) { + const log = { error: sinon.stub(), info: sinon.stub(), warn: sinon.stub() }; + const policyStore = { getDocument: sinon.stub().returns(document) }; + return { module: proxyquire(MODULE_PATH, { '../policyStore': policyStore, '../../lib/log': log }), log, policyStore }; +} + +describe('cpuThrottleExemption', () => { + afterEach(() => sinon.restore()); + + describe('when the policy is unknown', () => { + it('exempts nothing, so the throttle keeps its current behaviour', () => { + const { module: m } = load(null); + expect(m.getExemptPrefixes()).to.equal(null); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + }); + }); + + describe('when the document is malformed', () => { + it('exempts nothing and says so, rather than trusting part of it', () => { + const { module: m, log } = load(['enshrouded', 'Palworld']); + expect(m.getExemptPrefixes()).to.equal(null); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + expect(log.error.called).to.equal(true); + }); + + it('rejects a document that is not an array', () => { + const { module: m } = load({ enshrouded: true }); + expect(m.getExemptPrefixes()).to.equal(null); + }); + }); + + describe('when the bundle carries the document', () => { + it('exempts a hub game server: a listed prefix followed by its timestamp', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`palworld${TS}`, NOW)).to.equal(true); + }); + + it('compares case-insensitively, as app names are', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`PalWorld${TS}`, NOW)).to.equal(true); + }); + + it('reads a prefix that itself ends in a digit unambiguously', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`eurotrucksimulator2${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`7daystodie${TS}`, NOW)).to.equal(true); + }); + + it('matches the whole prefix, so one prefix never claims another product', () => { + const { module: m } = load(['rustserver']); + expect(m.isCpuThrottleExempt(`rustserver${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`rustserveroxide${TS}`, NOW)).to.equal(false); + }); + + it('does not exempt the prefix alone, or a name the hub did not mint', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt('enshrouded', NOW)).to.equal(false); + expect(m.isCpuThrottleExempt('enshroudedserver', NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(`enshrouded${TS}x`, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(`enshrouded_${TS}`, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(`myenshrouded${TS}`, NOW)).to.equal(false); + }); + + it('does not exempt a prefix the document does not list', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`wordpress${TS}`, NOW)).to.equal(false); + }); + + it('requires a plausible timestamp, not just thirteen digits', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt('enshrouded0000000000000', NOW), 'before the hub existed').to.equal(false); + expect(m.isCpuThrottleExempt('enshrouded9999999999999', NOW), 'centuries ahead').to.equal(false); + expect(m.isCpuThrottleExempt(`enshrouded${NOW + 2 * 24 * 60 * 60 * 1000}`, NOW), 'two days ahead').to.equal(false); + }); + + it('tolerates a browser clock running a little ahead of the node', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`enshrouded${NOW + 60 * 60 * 1000}`, NOW)).to.equal(true); + }); + + it('answers false for anything that is not a name', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(undefined, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(null, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(1792177003381, NOW)).to.equal(false); + }); + + it('treats an empty list as known, with nothing exempt', () => { + const { module: m } = load([]); + expect(m.getExemptPrefixes()).to.deep.equal(new Set()); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + }); + + it('rebuilds the set only when the store adopts a new bundle', () => { + const document = [...PREFIXES]; + const { module: m, policyStore } = load(document); + const first = m.getExemptPrefixes(); + expect(m.getExemptPrefixes()).to.equal(first); + + policyStore.getDocument.returns(['valheim']); + const second = m.getExemptPrefixes(); + expect(second).to.not.equal(first); + expect(m.isCpuThrottleExempt(`valheim${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + }); + }); +}); From b88ffd077a690544797c4cdbd307dc8c428e7627 Mon Sep 17 00:00:00 2001 From: Valter Silva Date: Sat, 17 Oct 2026 20:25:04 +0100 Subject: [PATCH 034/369] fix(apps): log a malformed exemption document once per bundle A malformed cputhrottleexempt document was re-validated, and logged as an error, for every app on every throttle pass. The verdict is now cached against the document like a valid one, and a valid bundle that replaces it is picked up as before. An exempt container already at its spec now logs that the throttle skipped it, as a burst-active one does, so a node log shows why a saturated game server was left alone. The module comment also names the dedicated sites' prefixes, which are not all marketplace names. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appManagement/appInspector.js | 2 ++ .../src/services/utils/cpuThrottleExemption.js | 14 +++++++++----- tests/unit/cpuThrottleExemption.test.js | 15 +++++++++++++++ 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/ZelBack/src/services/appManagement/appInspector.js b/ZelBack/src/services/appManagement/appInspector.js index cdbadf3a07..fa871845de 100644 --- a/ZelBack/src/services/appManagement/appInspector.js +++ b/ZelBack/src/services/appManagement/appInspector.js @@ -928,6 +928,8 @@ async function holdExemptAtSpecifiedCpu(containerName, specifiedCpu, inspect, mo if (nanoCpus && nanoCpus < specifiedNanoCpus) { await dockerService.appDockerUpdateCpu(containerName, specifiedNanoCpus); log.info(`checkApplicationsCpuUSage ${containerName} throttle-exempt, restoring cpu 100.`); + } else { + log.info(`checkApplicationsCpuUSage ${containerName} throttle-exempt, skipping CPU throttling`); } // Guarded for the same reason as the watermark writes in the caller. if (monitored) { diff --git a/ZelBack/src/services/utils/cpuThrottleExemption.js b/ZelBack/src/services/utils/cpuThrottleExemption.js index 98094264f7..5e62bc9513 100644 --- a/ZelBack/src/services/utils/cpuThrottleExemption.js +++ b/ZelBack/src/services/utils/cpuThrottleExemption.js @@ -2,10 +2,11 @@ const log = require('../../lib/log'); const policyStore = require('../policyStore'); // Which apps the CPU throttle leaves at their full specified allocation, read from the signed -// policy bundle. The document lists app name prefixes: the lowercased marketplace names of the -// game servers the games hub deploys. The hub names every server it registers -// ``, so a prefix alone is not enough to qualify -- the rest of -// the name must be exactly that millisecond timestamp, and a plausible one. +// policy bundle. The document lists app name prefixes: the ones the games hub, and the +// dedicated sites before it, write for the game servers they deploy (mostly the lowercased +// marketplace name). Every one of them names a server ``, so a prefix alone +// is not enough to qualify -- the rest of the name must be exactly that millisecond timestamp, +// and a plausible one. // // Game servers are exempt because a game's tick is a latency budget, not a throughput job. // Taking 20% of a saturated server's cpu does not slow it down evenly, it makes every tick that @@ -55,12 +56,15 @@ function getExemptPrefixes() { const document = policyStore.getDocument(DOCUMENT); if (document === null) return null; if (prefixCacheKey === document) return prefixCache; + // The verdict is cached either way. A malformed document is asked about once per app per + // throttle pass, and logging each time would bury the one line that matters. + prefixCacheKey = document; if (!isValidPrefixList(document)) { log.error(`cpuThrottleExemption - ${DOCUMENT} in the signed bundle is not an array of lowercase alphanumeric prefixes`); + prefixCache = null; return null; } prefixCache = new Set(document); - prefixCacheKey = document; return prefixCache; } diff --git a/tests/unit/cpuThrottleExemption.test.js b/tests/unit/cpuThrottleExemption.test.js index 976b4a8c6d..aa08cdc6f9 100644 --- a/tests/unit/cpuThrottleExemption.test.js +++ b/tests/unit/cpuThrottleExemption.test.js @@ -35,6 +35,21 @@ describe('cpuThrottleExemption', () => { expect(log.error.called).to.equal(true); }); + it('logs a malformed document once per bundle, not once per app', () => { + const { module: m, log } = load(['enshrouded', 'Palworld']); + m.isCpuThrottleExempt(`enshrouded${TS}`, NOW); + m.isCpuThrottleExempt(`palworld${TS}`, NOW); + m.isCpuThrottleExempt(`valheim${TS}`, NOW); + expect(log.error.callCount).to.equal(1); + }); + + it('recovers when the store adopts a valid bundle after a malformed one', () => { + const { module: m, policyStore } = load(['Palworld']); + expect(m.isCpuThrottleExempt(`palworld${TS}`, NOW)).to.equal(false); + policyStore.getDocument.returns(['palworld']); + expect(m.isCpuThrottleExempt(`palworld${TS}`, NOW)).to.equal(true); + }); + it('rejects a document that is not an array', () => { const { module: m } = load({ enshrouded: true }); expect(m.getExemptPrefixes()).to.equal(null); From b44aca6724b20538353db0f83c1d0e074d9a88c8 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:38:58 +0100 Subject: [PATCH 035/369] fix(upnp): a refusal for a port the router holds for something else is the port's, whatever the code After refusing a mapping the router is asked for its entry for the port. When it holds one that is not this node's mapping as asked - another machine's, or this node's to another port - the refusal names the port and does not count toward the router DOS, even when its code (501, 606) names nothing. A router that refuses everything with 501 otherwise counted every port another node behind it holds as the router failing. 728 says the table is full and counts whatever holds the port. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 10 ++++++-- tests/unit/upnpService.test.js | 38 +++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index dca0b90c1e..32a2be1555 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -62,8 +62,12 @@ const NO_SUCH_ENTRY = 714; // table is full: some answer 728 NoPortMapsAvailable, as the standard has it, // others a generic 501 Action Failed, which some also give for every refresh // of an entry they hold. A full table cannot be told by its code, so a refusal -// that is not the port's is taken as the router's. +// that is not the port's is taken as the router's - unless the router, asked +// after refusing, holds the port for something else: that names the port +// whatever the code, 501 and 606 included. Only 728 says the table is full in +// so many words, and counts as the router's even then. const PORT_REFUSALS = new Set([402, 716, 718, 724, 725, 726, 727, 729]); +const NO_PORT_MAPS_AVAILABLE = 728; // What a test leaves on the router while it runs and removes before it ends. const TEST_MAPPING_DESCRIPTIONS = new Set(['Flux_UPNP_Mapping_Test', 'Flux_Test_App']); @@ -126,16 +130,18 @@ async function mapProtocol(port, protocol, description) { return MAPPED; } catch (error) { const answered = error instanceof natUpnp.UpnpError; + let heldElsewhere = false; if (answered) { const held = await client.getMapping({ public: port, protocol }).catch(() => null); if (mapsHere(held, port)) { log.info(`UPnP - ${protocol} ${port} refused (${error.message}), but the router already maps it to this node`); return MAPPED; } + heldElsewhere = held !== null; } const code = answered ? error.code : null; // no answer, with no code, is the router's too - const routerWide = !PORT_REFUSALS.has(code); + const routerWide = !PORT_REFUSALS.has(code) && !(heldElsewhere && code !== NO_PORT_MAPS_AVAILABLE); if (routerWide) routerRefusalCount += 1; return { mapped: false, code, reason: `${protocol} ${port} not mapped: ${error.message}`, routerWide, diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 0b30e39ec1..3bd353264e 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -826,6 +826,44 @@ describe('upnpService tests', () => { }); }); + [ + [501, 'ActionFailed'], + [606, 'ActionNotAuthorized'], + ].forEach(([code, description]) => { + it(`should give a ${code} for a port another machine holds as the port's refusal, not the router's`, async () => { + createMappingStub.rejects(refusal(code, description)); + getMappingStub.resolves(entry({ local: false, private: { host: '192.168.1.11', port: 123 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); + }); + + it('should give a 501 for a port this node holds otherwise as the port\'s refusal, not the router\'s', async () => { + createMappingStub.rejects(refusal(501, 'ActionFailed')); + getMappingStub.resolves(entry({ private: { host: '192.168.1.10', port: 124 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 501, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); + + it('should count a 728 as the router\'s refusal even for a port another machine holds', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.resolves(entry({ local: false, private: { host: '192.168.1.11', port: 123 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 728, routerWide: true }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + }); + it('should take a refusal as mapped when the router already maps the port to this node', async () => { createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); getMappingStub.callsFake(async ({ protocol }) => entry({ protocol: protocol.toLowerCase() })); From 495e70b962e0b71988867ae4d54920a94f34d2f9 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:55:46 +0100 Subject: [PATCH 036/369] fix(upnp): remove only the protocols of a port this node mapped, without asking whose they are mapUpnpPort records, per protocol, whether its latest map of a port left the router mapping it here: a mapping the router took, or a refusal for an entry it already holds for this node. removeMapUpnpPort removes exactly those protocols and asks the router nothing first. A router refuses to map a port another machine holds, so what it took is this node's and what it refused never is: a test port another node holds is never asked to be removed, a TCP half kept after a refused UDP is, and a lookup that fails can no longer turn into a removal by port of someone else's mapping. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 72 +++++++++++++--------- tests/unit/upnpService.test.js | 95 ++++++++++++++++++++--------- 2 files changed, 109 insertions(+), 58 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 32a2be1555..bc76c42b73 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -38,6 +38,26 @@ const OWNED_MAPPING_PREFIXES = ['Flux_App_', 'Flux_Prelaunch_App_']; // port mapped here is in use - a test port mid-test, an app installing - // whatever the app table says so far. const mappedHere = new Set(); +// The protocols of each port whose latest map here created the mapping or found +// it already this node's, by port: what removeMapUpnpPort removes. A router +// refuses to map a port another machine holds, so a mapping it took is this +// node's, and one it refused never is. +const mappedProtocols = new Map(); + +/** + * Records whether this node's latest map of one protocol of a port left the + * router mapping it here. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @param {boolean} mapped + */ +function recordMapped(port, protocol, mapped) { + const protocols = mappedProtocols.get(port) || new Set(); + if (mapped) protocols.add(protocol); + else protocols.delete(protocol); + if (protocols.size) mappedProtocols.set(port, protocols); + else mappedProtocols.delete(port); +} // The cleanup pass runs in a wall-clock slot of its own, laid out as #1710 laid // out its refresh: a 10-minute cycle of eight 75-second slots, the slot chosen // by the api port's tens digit. Nodes behind one public address have distinct @@ -367,9 +387,8 @@ async function setupUPNP(apiport = config.server.apiport) { /** * Maps a port to this node over TCP and then UDP. * - * On a failure the port is no longer held here: a TCP half the router kept is - * the caller's to remove with removeMapUpnpPort, or the stale mapping - * cleanup's. + * On a failure the port is no longer held here; a TCP half the router took is + * still recorded, for removeMapUpnpPort to remove. * @param {number} port Port number. * @param {string} description Port description. * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} @@ -380,9 +399,11 @@ async function mapUpnpPort(port, description) { // moment it is there mappedHere.add(+port); let result = await mapProtocol(+port, 'TCP', description); + recordMapped(+port, 'TCP', result.mapped); if (result.mapped) { await serviceHelper.delay(500); result = await mapProtocol(+port, 'UDP', description); + recordMapped(+port, 'UDP', result.mapped); } if (!result.mapped) { log.error(`UPnP - ${result.reason}`); @@ -394,28 +415,12 @@ async function mapUpnpPort(port, description) { } /** - * Removes the router's mapping of one protocol of a port when it points at - * this node. A router removes a mapping by its external port alone, whoever it - * points at, so the entry is looked up first: one pointing at another machine - * - another node behind the same router holding that port - is left, and one - * the router does not hold is gone already. An entry that cannot be looked up - * is removed. + * Removes the router's mapping of one protocol of a port. * @param {number} port * @param {string} protocol 'TCP' or 'UDP' - * @returns {Promise} whether no mapping of it to this node is left + * @returns {Promise} whether the router no longer holds it */ async function removeProtocol(port, protocol) { - let held; - try { - held = await client.getMapping({ public: port, protocol }); - } catch (error) { - log.warn(`UPnP - ${protocol} ${port} not looked up, removing it: ${error.message}`); - } - if (held === null) return true; - if (held && held.local === false) { - log.info(`UPnP - ${protocol} ${port} left mapped: the router maps it to ${held.private.host}`); - return true; - } try { await client.removeMapping({ public: port, protocol }); return true; @@ -427,19 +432,29 @@ async function removeProtocol(port, protocol) { } /** - * Removes the TCP and the UDP mapping of a port to this node, each on its own: - * a refusal of one leaves the other to go. + * Removes the protocols of a port that this node mapped, each on its own: a + * refusal of one leaves the other to go. A router removes a mapping by its + * external port whoever it points at, so a protocol this node's latest map did + * not take - one another machine holds - is never asked for. * @param {number} port Port number. * @returns {Promise} whether no mapping of the port to this node is left */ async function removeMapUpnpPort(port) { + const protocols = mappedProtocols.get(+port) || new Set(); // given up even if the router refuses: what is left is the cleanup pass's mappedHere.delete(+port); - const tcpRemoved = await removeProtocol(+port, 'TCP'); - await serviceHelper.delay(500); - const udpRemoved = await removeProtocol(+port, 'UDP'); - await serviceHelper.delay(500); - return tcpRemoved && udpRemoved; + mappedProtocols.delete(+port); + let removed = true; + // eslint-disable-next-line no-restricted-syntax + for (const protocol of ['TCP', 'UDP']) { + // eslint-disable-next-line no-continue + if (!protocols.has(protocol)) continue; + // eslint-disable-next-line no-await-in-loop + if (!await removeProtocol(+port, protocol)) removed = false; + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(500); + } + return removed; } /** @@ -762,6 +777,7 @@ module.exports = { msUntilCleanupSlot, removeStaleMappings, mappedHere, + mappedProtocols, CLEANUP_SLOT_MS, mapPortApi, removeMapPortApi, diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 3bd353264e..91a970a599 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -929,26 +929,20 @@ describe('upnpService tests', () => { describe('removeMapUpnpPort tests', () => { let logSpy; + let createMappingStub; let getMappingStub; let removeMappingStub; - const entry = (protocol, overrides = {}) => ({ - public: { host: '', port: 123 }, - private: { host: '192.168.1.10', port: 123 }, - protocol, - enabled: true, - description: 'Flux_App_web', - ttl: 0, - local: true, - ...overrides, - }); + const refusal = (code, description, action) => new natUpnp.UpnpError(code, description, action); const removedProtocols = () => removeMappingStub.getCalls().map((call) => call.args[0].protocol); beforeEach(() => { + upnpService.mappedHere.clear(); + upnpService.mappedProtocols.clear(); logSpy = sinon.spy(log, 'error'); sinon.stub(serviceHelper, 'delay').resolves(); - getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping') - .callsFake(async ({ protocol }) => entry(protocol.toLowerCase())); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); @@ -956,47 +950,77 @@ describe('upnpService tests', () => { sinon.restore(); }); - it('should remove both protocols of a port the router maps to this node', async () => { + it('should remove both protocols this node mapped, without asking the router whose they are', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + getMappingStub.resetHistory(); + const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(true); - sinon.assert.notCalled(logSpy); - sinon.assert.calledWithExactly(getMappingStub, { public: 123, protocol: 'TCP' }); - sinon.assert.calledWithExactly(getMappingStub, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(getMappingStub); sinon.assert.calledWithExactly(removeMappingStub.firstCall, { public: 123, protocol: 'TCP' }); sinon.assert.calledWithExactly(removeMappingStub.secondCall, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(logSpy); }); - it('should leave a protocol the router maps to another machine', async () => { - getMappingStub.withArgs({ public: 123, protocol: 'UDP' }) - .resolves(entry('udp', { local: false, private: { host: '192.168.1.11', port: 123 } })); + it('should remove the TCP half of a port whose UDP the router refused', async () => { + createMappingStub.onSecondCall().rejects(refusal(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); - const result = await upnpService.removeMapUpnpPort(123); + await upnpService.removeMapUpnpPort(123); - expect(result).to.equal(true); expect(removedProtocols()).to.deep.equal(['TCP']); }); - it('should remove nothing of a protocol the router does not hold', async () => { - getMappingStub.withArgs({ public: 123, protocol: 'TCP' }).resolves(null); + it('should ask the router to remove nothing of a port it refused to map here', async () => { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(true); - expect(removedProtocols()).to.deep.equal(['UDP']); + sinon.assert.notCalled(removeMappingStub); }); - it('should remove a protocol whose entry cannot be looked up', async () => { - getMappingStub.rejects(new Error('socket hang up')); - + it('should ask the router to remove nothing of a port this node never mapped', async () => { const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(true); + sinon.assert.notCalled(removeMappingStub); + }); + + it('should forget a protocol whose latest map the router refused', async () => { + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + await upnpService.removeMapUpnpPort(123); + + // TCP was refused; UDP, not asked again, stays as the first map left it + expect(removedProtocols()).to.deep.equal(['UDP']); + }); + + it('should count a refusal the router has the port mapped here for as mapped here', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable', 'AddPortMapping')); + getMappingStub.callsFake(async ({ protocol }) => ({ + public: { host: '', port: 123 }, + private: { host: '192.168.1.10', port: 123 }, + protocol: protocol.toLowerCase(), + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + })); + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + await upnpService.removeMapUpnpPort(123); + expect(removedProtocols()).to.deep.equal(['TCP', 'UDP']); }); - it('should remove UDP when TCP is refused, and report the port not removed', async () => { - removeMappingStub.withArgs({ public: 123, protocol: 'TCP' }).rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping')); + it('should remove UDP when the router refuses to remove TCP, and report the port not removed', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + removeMappingStub.withArgs({ public: 123, protocol: 'TCP' }).rejects(refusal(501, 'ActionFailed', 'DeletePortMapping')); const result = await upnpService.removeMapUpnpPort(123); @@ -1006,13 +1030,24 @@ describe('upnpService tests', () => { }); it('should take a removal the router has no entry for as removed', async () => { - removeMappingStub.rejects(new natUpnp.UpnpError(714, 'NoSuchEntryInArray', 'DeletePortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + removeMappingStub.rejects(refusal(714, 'NoSuchEntryInArray', 'DeletePortMapping')); const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(true); sinon.assert.notCalled(logSpy); }); + + it('should ask a second removal of the same port for nothing', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + await upnpService.removeMapUpnpPort(123); + removeMappingStub.resetHistory(); + + await upnpService.removeMapUpnpPort(123); + + sinon.assert.notCalled(removeMappingStub); + }); }); describe('msUntilCleanupSlot tests', () => { From bea7ef5ce47430ef4febd6e057df78183db25963 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 12:06:43 +0100 Subject: [PATCH 037/369] fix(upnp): a removal the router refuses is asked again until it goes, and no test mapping is added while one waits removeMapUpnpPort keeps each removal of this node's own mapping that the router refuses, and asks again after config.upnp.removalRetryFirstMs (30 s), the wait doubling to config.upnp.removalRetryMaxMs (30 min), until the router removes it or holds no such entry. Pending removals are asked one at a time. A port this node maps again is in use, and its pending removal is dropped. While a removal of a test mapping is pending, the availability check adds no new test mapping, so a router refusing removals gathers no more of them. verifyUPNPsupport removes its test mapping through removeMapUpnpPort: once the router has taken the mapping and shown it, UPnP works, and a removal it refuses is pending rather than a failed verification. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 4 + .../appMonitoring/availabilityChecker.js | 9 + ZelBack/src/services/upnpService.js | 97 ++++++++-- tests/unit/availabilityChecker.test.js | 12 ++ tests/unit/globalconfig/default.js | 2 + tests/unit/upnpService.test.js | 169 ++++++++++++++++-- 6 files changed, 263 insertions(+), 30 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index f7b06e863e..ba45df6173 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -107,6 +107,10 @@ module.exports = { upnp: { gatewayUrl: '', nodeIp: '', + // a removal of this node's own mapping the router refuses is asked again + // after this long, the wait doubling up to removalRetryMaxMs + removalRetryFirstMs: 30 * 1000, + removalRetryMaxMs: 30 * 60 * 1000, }, benchmark: { host: '127.0.0.1', diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index de3fb93891..e74a99d952 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -237,6 +237,15 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } } + // A router that refused to remove the last test mapping is not given + // another until it has. + if (isUpnp && upnpService.testRemovalPending()) { + log.info('checkMyAppsAvailability - a test mapping is still to be removed from the router; no new one until it is'); + await serviceHelper.delay(timeouts.error); + setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); + return; + } + if (appPorts.includes(dosState.testingPort)) { log.info(`checkMyAppsAvailability - Skipped checking ${dosState.testingPort} - in use`); setNextPort(); diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index bc76c42b73..80f832b340 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -44,6 +44,14 @@ const mappedHere = new Set(); // node's, and one it refused never is. const mappedProtocols = new Map(); +// Removals of this node's own mappings the router refused, by `port/protocol`: +// each is asked again, one at a time, after config.upnp.removalRetryFirstMs, +// the wait doubling to config.upnp.removalRetryMaxMs, until the router removes +// it or holds no such entry. A port this node maps again is in use, and its +// pending removal is dropped. +const pendingRemovals = new Map(); +let pendingRemovalTurn = Promise.resolve(); + /** * Records whether this node's latest map of one protocol of a port left the * router mapping it here. @@ -280,6 +288,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { upnpMachine = false; return false; } + recordMapped(+apiport + 3, 'TCP', true); await serviceHelper.delay(500); try { const mapping = await client.getMapping({ public: +apiport + 3, protocol: 'TCP' }); @@ -292,19 +301,9 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { upnpMachine = false; return false; } - try { - await client.removeMapping({ - public: +apiport + 3, - }); - - await serviceHelper.delay(500); - } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed Remove Mapping'); - upnpMachine = false; - return false; - } - mappedHere.delete(+apiport + 3); + // UPnP works once the router has taken the mapping and shows it: a removal it + // refuses is asked again until it goes, and fails nothing here. + await removeMapUpnpPort(+apiport + 3, 'Flux_UPNP_Mapping_Test'); upnpMachine = true; return true; @@ -398,6 +397,7 @@ async function mapUpnpPort(port, description) { // held from before the router has it: the cleanup pass may list it the // moment it is there mappedHere.add(+port); + dropPendingRemovals(+port); let result = await mapProtocol(+port, 'TCP', description); recordMapped(+port, 'TCP', result.mapped); if (result.mapped) { @@ -431,17 +431,68 @@ async function removeProtocol(port, protocol) { } } +/** + * Drops the pending removals of a port. + * @param {number} port + */ +function dropPendingRemovals(port) { + ['TCP', 'UDP'].forEach((protocol) => { + const pending = pendingRemovals.get(`${port}/${protocol}`); + if (!pending) return; + clearTimeout(pending.timer); + pendingRemovals.delete(`${port}/${protocol}`); + }); +} + +/** + * Asks the router once more to remove a pending removal's mapping, in turn + * behind any other pending removal, and waits longer before the next ask if it + * refuses again. + * @param {object} pending + */ +function schedulePendingRemoval(pending) { + const { removalRetryFirstMs, removalRetryMaxMs } = config.upnp; + const waitMs = Math.min(removalRetryFirstMs * (2 ** pending.refusals), removalRetryMaxMs); + // eslint-disable-next-line no-param-reassign + pending.refusals += 1; + // eslint-disable-next-line no-param-reassign + pending.timer = setTimeout(() => { + pendingRemovalTurn = pendingRemovalTurn.then(async () => { + const key = `${pending.port}/${pending.protocol}`; + if (pendingRemovals.get(key) !== pending) return; + if (await removeProtocol(pending.port, pending.protocol)) { + pendingRemovals.delete(key); + log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); + return; + } + // mapped again while the router was being asked: in use, not pending + if (pendingRemovals.get(key) === pending) schedulePendingRemoval(pending); + }); + }, waitMs); +} + +/** + * Whether a removal of one of this node's test mappings is still pending: no + * new test mapping is made until it has gone, so a router refusing removals + * gathers no more of them. + * @returns {boolean} + */ +function testRemovalPending() { + return [...pendingRemovals.values()].some((pending) => isTestMappingDescription(pending.description)); +} + /** * Removes the protocols of a port that this node mapped, each on its own: a * refusal of one leaves the other to go. A router removes a mapping by its * external port whoever it points at, so a protocol this node's latest map did - * not take - one another machine holds - is never asked for. + * not take - one another machine holds - is never asked for. A removal the + * router refuses is pending, and asked again until it goes. * @param {number} port Port number. - * @returns {Promise} whether no mapping of the port to this node is left + * @param {string} [description] the mapping's description, for the log and testRemovalPending + * @returns {Promise} whether the router removed every one of them now */ -async function removeMapUpnpPort(port) { +async function removeMapUpnpPort(port, description = '') { const protocols = mappedProtocols.get(+port) || new Set(); - // given up even if the router refuses: what is left is the cleanup pass's mappedHere.delete(+port); mappedProtocols.delete(+port); let removed = true; @@ -450,7 +501,15 @@ async function removeMapUpnpPort(port) { // eslint-disable-next-line no-continue if (!protocols.has(protocol)) continue; // eslint-disable-next-line no-await-in-loop - if (!await removeProtocol(+port, protocol)) removed = false; + if (!await removeProtocol(+port, protocol)) { + removed = false; + const pending = { + port: +port, protocol, description, refusals: 0, timer: null, + }; + dropPendingRemovals(+port); + pendingRemovals.set(`${+port}/${protocol}`, pending); + schedulePendingRemoval(pending); + } // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(500); } @@ -778,6 +837,8 @@ module.exports = { removeStaleMappings, mappedHere, mappedProtocols, + pendingRemovals, + testRemovalPending, CLEANUP_SLOT_MS, mapPortApi, removeMapPortApi, diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index bae90ed14c..d97a2f67ca 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -361,6 +361,7 @@ describe('availabilityChecker tests', () => { sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); sinon.stub(upnpService, 'routerRefusals').callsFake(() => refusals); + sinon.stub(upnpService, 'testRemovalPending').returns(false); mapUpnpPortStub = sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); removeMapUpnpPortStub = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { @@ -404,6 +405,17 @@ describe('availabilityChecker tests', () => { expect(mockDosState.portFailureScore ?? 0).to.equal(0); }); + it('makes no test mapping, and opens no port, while the router has yet to remove the last one', async () => { + upnpService.testRemovalPending.returns(true); + + await pass(); + + sinon.assert.notCalled(mapUpnpPortStub); + sinon.assert.notCalled(fluxNetworkHelper.allowTestPort); + sinon.assert.notCalled(portTestStub); + expect(mockDosState.portFailureScore ?? 0, 'counts nothing').to.equal(0); + }); + it('removes what the router kept of a test mapping it refused', async () => { testMappingRefused(refused(718, false)); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 4ca46c9505..637c92bc23 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -87,6 +87,8 @@ module.exports = { upnp: { gatewayUrl: '', nodeIp: '', + removalRetryFirstMs: 30 * 1000, + removalRetryMaxMs: 30 * 60 * 1000, }, benchmark: { host: '127.0.0.1', diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 91a970a599..e8ef9357ba 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -162,14 +162,14 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); }); - it('should hold the test port from the time it is mapped until it is removed, so a cleanup pass cannot take it mid-test', async () => { + it('should hold the test port from the time it is mapped until it is read back, so a cleanup pass cannot take it mid-test', async () => { upnpService.mappedHere.clear(); const heldAt = {}; sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); sinon.stub(natUpnp.Client.prototype, 'createMapping').callsFake(async () => { heldAt.create = upnpService.mappedHere.has(16130); }); - sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); - sinon.stub(natUpnp.Client.prototype, 'removeMapping').callsFake(async () => { heldAt.remove = upnpService.mappedHere.has(16130); }); + sinon.stub(natUpnp.Client.prototype, 'getMapping').callsFake(async () => { heldAt.readBack = upnpService.mappedHere.has(16130); return testMapping; }); + const removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); const clock = sinon.useFakeTimers(); @@ -178,7 +178,8 @@ describe('upnpService tests', () => { await clock.tickAsync(2_500); expect(await promise).to.equal(true); - expect(heldAt).to.deep.equal({ create: true, remove: true }); + expect(heldAt).to.deep.equal({ create: true, readBack: true }); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 16130, protocol: 'TCP' }); expect(upnpService.mappedHere.has(16130)).to.equal(false); }); @@ -451,12 +452,14 @@ describe('upnpService tests', () => { sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mapping'); }); - it('should log a proper error if removeMapping throws', async () => { - sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); + it('should verify when the router refuses to remove the test mapping, and leave its removal pending', async () => { + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); - sinon.stub(natUpnp.Client.prototype, 'removeMapping').throws(); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping')); const clock = sinon.useFakeTimers(); @@ -465,9 +468,11 @@ describe('upnpService tests', () => { await clock.tickAsync(2_500); const result = await promise; - expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Remove Mapping'); + expect(result).to.equal(true); + expect([...upnpService.pendingRemovals.keys()]).to.deep.equal(['16130/TCP']); + expect(upnpService.testRemovalPending()).to.equal(true); + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); }); }); @@ -1050,6 +1055,146 @@ describe('upnpService tests', () => { }); }); + describe('pending removal tests', () => { + const FIRST_MS = 30 * 1000; + const MAX_MS = 30 * 60 * 1000; + let clock; + let createMappingStub; + let removeMappingStub; + + const refused = () => new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping'); + const removals = () => removeMappingStub.getCalls().map((call) => `${call.args[0].public}/${call.args[0].protocol}`); + // a test port mapped TCP only: its UDP refused + const mapTcpOnly = async (port, description) => { + createMappingStub.withArgs(sinon.match({ public: port, protocol: 'UDP' })).rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(port, description); + }; + + beforeEach(() => { + upnpService.mappedHere.clear(); + upnpService.mappedProtocols.clear(); + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); + sinon.stub(serviceHelper, 'delay').resolves(); + sinon.stub(log, 'error'); + sinon.stub(log, 'info'); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); + removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + clock = sinon.useFakeTimers(); + }); + + afterEach(() => { + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); + clock.restore(); + sinon.restore(); + }); + + it('should ask the router again for a removal it refused, waiting longer each time, until it goes', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + removeMappingStub.onCall(2).rejects(refused()); + + expect(await upnpService.removeMapUpnpPort(123, 'Flux_Test_App')).to.equal(false); + expect(removals()).to.deep.equal(['123/TCP']); + + await clock.tickAsync(FIRST_MS - 1); + expect(removals().length, 'nothing before the first wait').to.equal(1); + await clock.tickAsync(1); + expect(removals().length, 'asked again after the first wait').to.equal(2); + await clock.tickAsync(2 * FIRST_MS - 1); + expect(removals().length, 'the wait doubles').to.equal(2); + await clock.tickAsync(1); + expect(removals().length).to.equal(3); + await clock.tickAsync(4 * FIRST_MS); + expect(removals().length, 'removed on the fourth ask').to.equal(4); + expect(upnpService.pendingRemovals.size).to.equal(0); + + await clock.tickAsync(MAX_MS * 2); + expect(removals().length, 'nothing once removed').to.equal(4); + }); + + it('should never give up, and wait no longer than the cap between asks', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + // 30s, 1m, 2m, 4m, 8m, 16m, then 30m each + await clock.tickAsync((30 + 60 + 120 + 240 + 480 + 960) * 1000); + expect(removals().length).to.equal(7); + await clock.tickAsync(MAX_MS - 1); + expect(removals().length).to.equal(7); + await clock.tickAsync(1); + expect(removals().length).to.equal(8); + await clock.tickAsync(MAX_MS * 10); + expect(removals().length, 'still asking').to.equal(18); + expect(upnpService.pendingRemovals.has('123/TCP')).to.equal(true); + }); + + it('should take a router that holds no such entry as removed', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(new natUpnp.UpnpError(714, 'NoSuchEntryInArray', 'DeletePortMapping')); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + await clock.tickAsync(FIRST_MS); + expect(upnpService.pendingRemovals.size).to.equal(0); + await clock.tickAsync(MAX_MS); + expect(removals().length).to.equal(2); + }); + + it('should drop a pending removal of a port this node maps again', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + await clock.tickAsync(MAX_MS); + + expect(removals()).to.deep.equal(['123/TCP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should ask for pending removals one at a time', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + await mapTcpOnly(124, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + await upnpService.removeMapUpnpPort(124, 'Flux_Test_App'); + let inFlight = 0; + let most = 0; + removeMappingStub.callsFake(async () => { + inFlight += 1; + most = Math.max(most, inFlight); + await new Promise((resolve) => { setTimeout(resolve, 1000); }); + inFlight -= 1; + }); + + await clock.tickAsync(FIRST_MS + 5000); + + expect(removals().slice(2).sort()).to.deep.equal(['123/TCP', '124/TCP']); + expect(most, 'never two at once').to.equal(1); + }); + + it('should hold back new test mappings while a test mapping\'s removal is pending, and only then', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + expect(upnpService.testRemovalPending(), 'an app mapping holds nothing back').to.equal(false); + + await mapTcpOnly(124, 'Flux_Test_App'); + removeMappingStub.onCall(1).rejects(refused()); + await upnpService.removeMapUpnpPort(124, 'Flux_Test_App'); + expect(upnpService.testRemovalPending()).to.equal(true); + + await clock.tickAsync(FIRST_MS); + expect(upnpService.testRemovalPending(), 'removed on the next ask').to.equal(false); + }); + }); + describe('msUntilCleanupSlot tests', () => { const at = (cycleOffsetMs) => 1_800_000_000_000 - (1_800_000_000_000 % (10 * 60 * 1000)) + cycleOffsetMs; From e9d4f0a69944acdf72e95a4ba913ddfd3ec249b1 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 12:06:43 +0100 Subject: [PATCH 038/369] test(harness): 3002 a node asks again for a removal its router refused The stub gateway refuses the first `refuseDeletes` DeletePortMapping requests with 501 and marks each refused request in its state. The harness asks a refused removal again after 2 s, the wait doubling to 8 s. Co-Authored-By: Claude Opus 5.5 --- test-infra/config/shared.js | 6 +- test-infra/external-http-stub/index.js | 10 ++- .../runner/framework/external-http-control.js | 8 +- ...-again-for-a-removal-its-router-refused.js | 76 +++++++++++++++++++ 4 files changed, 95 insertions(+), 5 deletions(-) create mode 100644 test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js diff --git a/test-infra/config/shared.js b/test-infra/config/shared.js index ab8ca5166c..8894091630 100644 --- a/test-infra/config/shared.js +++ b/test-infra/config/shared.js @@ -12,7 +12,11 @@ module.exports = { // the behaviour is unchanged, only the searching stops. // nodeIp stays empty as in production: it is the node's own address for a port mapping, // it cannot be a shared constant, and no mapping is ever made because verification fails. - upnp: { gatewayUrl: 'http://31.200.0.6:3000/upnp/device.xml', nodeIp: '' }, + // A removal the router refuses is asked again after 2s, the wait doubling to 8s, so a + // suite sees several asks in seconds rather than the half hour production allows. + upnp: { + gatewayUrl: 'http://31.200.0.6:3000/upnp/device.xml', nodeIp: '', removalRetryFirstMs: 2000, removalRetryMaxMs: 8000, + }, // Empty disables analytics. The app default is the live cloudaudit endpoint, which // the fleet must never report suite activity to. analytics: { url: '' }, diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index eafa884f69..af0f03250d 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -796,6 +796,12 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { return; } case 'DeletePortMapping': { + if (gateway.refuseDeletes > 0) { + gateway.refuseDeletes -= 1; + gateway.requests[gateway.requests.length - 1].refused = 501; + soapFault(res, 501, 'ActionFailed'); + return; + } const before = gateway.mappings.length; gateway.mappings = gateway.mappings.filter((m) => !sameEntry(m)); if (gateway.mappings.length === before) soapFault(res, 714, 'NoSuchEntryInArray'); @@ -1012,14 +1018,14 @@ control.post('/storage-requests/reset', (req, res) => { // Put a node behind a UPnP gateway, with no mappings yet. control.post('/upnp-gateway', (req, res) => { const { - ip, externalIp = null, capacity = null, mappings = [], + ip, externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, } = req.body || {}; if (!ip) { res.status(400).json({ error: 'ip is required' }); return; } state.upnpGateways[ip] = { - externalIp, capacity, mappings, requests: [], + externalIp, capacity, mappings, refuseDeletes, requests: [], }; res.json({ ok: true }); }); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index 79a020205b..62acf74a0f 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -195,10 +195,14 @@ export async function resetStorageRequests() { * @param {string} [opts.externalIp] - the address the gateway reports; the node's own when absent * @param {number} [opts.capacity] - how many mappings its table holds; no limit when absent * @param {object[]} [opts.mappings] - what its table holds already, as upnpGatewayState lists it + * @param {number} [opts.refuseDeletes] - how many DeletePortMapping requests it refuses with + * 501 before it removes anything; a refused request carries `refused: 501` in its state */ -export async function enableUpnpGateway(ip, { externalIp = null, capacity = null, mappings = [] } = {}) { +export async function enableUpnpGateway(ip, { + externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, +} = {}) { const res = await post('/upnp-gateway', { - ip, externalIp, capacity, mappings, + ip, externalIp, capacity, mappings, refuseDeletes, }); if (!res.ok) throw new Error(`enableUpnpGateway: ${JSON.stringify(res)}`); } diff --git a/test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js b/test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js new file mode 100644 index 0000000000..94f55430f4 --- /dev/null +++ b/test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js @@ -0,0 +1,76 @@ +// weight: light +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const subnet = getSubnetConfig(); + +// A node verifies UPnP by mapping a test port, reading it back and removing it. Its router +// refuses the first removals it is asked for. The node verifies all the same - the router +// took the mapping and showed it - and asks again for the removal, waiting longer each +// time (test-infra/config/shared.js: 2s, doubling to 8s), until the router removes it. + +const API_PORT = 16137; +const TEST_PORT = API_PORT + 3; +const REFUSED = 2; +const FIRST_RETRY_MS = 2000; + +describe('a node asks again for a removal its router refused', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); + await enableUpnpGateway(nodeIp, { refuseDeletes: REFUSED }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('verifies, and removes its test mapping once the router takes the removal', async function () { + this.timeout(180000); + const corePorts = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2]; + let state = { mappings: [], requests: [] }; + const removalsOfTestPort = () => state.requests.filter((r) => r.action === 'DeletePortMapping' && r.port === TEST_PORT); + await waitFor(async () => { + state = await upnpGatewayState(nodeIp); + return removalsOfTestPort().length > REFUSED + && corePorts.every((port) => state.mappings.some((m) => m.externalPort === port)); + }, { timeout: 150000, interval: 1000, label: 'the test mapping removed and the core ports mapped' }).catch(() => {}); + + const asks = removalsOfTestPort(); + expect(asks.filter((r) => r.refused === 501).length, 'canary: the router refused the first removals').to.equal(REFUSED); + expect(asks.length, 'asked again until the router took it').to.equal(REFUSED + 1); + // a node that failed verification never maps its own ports + expect(corePorts.every((port) => state.mappings.some((m) => m.externalPort === port)), 'verified: its own ports mapped').to.equal(true); + expect(state.mappings.some((m) => m.externalPort === TEST_PORT), 'the test mapping is gone').to.equal(false); + + const gaps = asks.slice(1).map((r, i) => Date.parse(r.at) - Date.parse(asks[i].at)); + expect(gaps[0], 'the first ask again waits the first retry').to.be.at.least(FIRST_RETRY_MS - 250); + expect(gaps[1], 'the next waits twice as long').to.be.at.least(2 * FIRST_RETRY_MS - 250); + }); + + it('asks for no other removal of the test port', async function () { + this.timeout(30000); + await new Promise((resolve) => { setTimeout(resolve, 10000); }); + const { requests } = await upnpGatewayState(nodeIp); + expect(requests.filter((r) => r.action === 'DeletePortMapping' && r.port === TEST_PORT).length).to.equal(REFUSED + 1); + }); +}); From 75f57432cfb3b4ecd47331aed0857969d972ef53 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 13:35:58 +0100 Subject: [PATCH 039/369] fix(upnp): a node verifying UPnP on a full table waits its slot's turn before listing it verifyUPNPsupport lists the router's table at once when the test mapping is refused for a reason of the router's own: the node cannot run until it has verified, so it cannot wait for its cleanup slot. It now waits its slot's number times 5 s first, so nodes behind one router starting together list it in turn. A router serves one request at a time: six nodes listing a 138-entry table at once took 23 to 38 s each, where one alone took 0.2 s, against the cleanup's 60 s listing budget. A router with room for the test mapping is never listed and never waited for. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 20 ++++++++++++++++- tests/unit/upnpService.test.js | 33 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 80f832b340..04bb5ab0bb 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -77,6 +77,13 @@ const CLEANUP_SLOT_MS = CLEANUP_CYCLE_MS / CLEANUP_SLOT_COUNT; // The listing has to finish this long before the slot ends, so that what it // lists still has time to be removed. const CLEANUP_REMOVAL_RESERVE_MS = 15 * 1000; +// A node verifying UPnP on a full table lists it at once, without waiting for +// its slot: it cannot run until it has verified. It waits its slot's number +// times this first, so siblings starting together - a power cut, a router +// restart, one update reaching them all - take turns: a router serves one +// request at a time, and six listings at once took 23 to 38 s each where one +// alone took 0.2 s. +const BOOT_LISTING_STAGGER_MS = 5 * 1000; const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); @@ -263,6 +270,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { // not listed. if (testMapping.routerWide && testMapping.code !== null) { let removed = 0; + await serviceHelper.delay(cleanupSlotOf(apiport) * BOOT_LISTING_STAGGER_MS); try { removed = await removeLeftoverTestMappings(); } catch (error) { @@ -548,6 +556,15 @@ function corePorts(apiport) { return [+apiport - 1, +apiport, +apiport + 1, +apiport + 2, fluxadmPort.sshPortFor(apiport)]; } +/** + * The number of this node's cleanup slot, from its api port's tens digit. + * @param {number|string} apiport the api port this FluxOS listens on + * @returns {number} 0 to CLEANUP_SLOT_COUNT - 1 + */ +function cleanupSlotOf(apiport) { + return Math.floor((+apiport % 100) / 10) % CLEANUP_SLOT_COUNT; +} + /** * Milliseconds until the next start of this node's cleanup slot. * @param {number|string} apiport the api port this FluxOS listens on @@ -555,7 +572,7 @@ function corePorts(apiport) { * @returns {number} */ function msUntilCleanupSlot(apiport, now = Date.now()) { - const slotStart = (Math.floor((+apiport % 100) / 10) % CLEANUP_SLOT_COUNT) * CLEANUP_SLOT_MS; + const slotStart = cleanupSlotOf(apiport) * CLEANUP_SLOT_MS; return (slotStart - (now % CLEANUP_CYCLE_MS) + CLEANUP_CYCLE_MS) % CLEANUP_CYCLE_MS; } @@ -840,6 +857,7 @@ module.exports = { pendingRemovals, testRemovalPending, CLEANUP_SLOT_MS, + BOOT_LISTING_STAGGER_MS, mapPortApi, removeMapPortApi, getMapApi, diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index e8ef9357ba..00c38dff68 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -162,6 +162,21 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); }); + it('should not wait for a turn when the router has room for the test mapping', async () => { + const delayStub = sinon.stub(serviceHelper, 'delay').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + const getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + + expect(await upnpService.verifyUPNPsupport(16177)).to.equal(true); + + expect(delayStub.getCalls().every((call) => call.args[0] < upnpService.BOOT_LISTING_STAGGER_MS)).to.equal(true); + sinon.assert.notCalled(getMappingsStub); + }); + it('should hold the test port from the time it is mapped until it is read back, so a cleanup pass cannot take it mid-test', async () => { upnpService.mappedHere.clear(); const heldAt = {}; @@ -285,6 +300,24 @@ describe('upnpService tests', () => { sinon.assert.calledTwice(createMappingStub); }); + [ + [16127, 2], + [16137, 3], + [16177, 7], + [16187, 0], + [16197, 1], + ].forEach(([apiport, slot]) => { + it(`should wait slot ${slot}'s turn before listing the table on api port ${apiport}`, async () => { + getMappingsStub.resolves([]); + + await upnpService.verifyUPNPsupport(apiport); + + const wait = serviceHelper.delay.withArgs(slot * upnpService.BOOT_LISTING_STAGGER_MS); + sinon.assert.called(wait); + expect(wait.calledBefore(getMappingsStub), 'the wait comes before the listing').to.equal(true); + }); + }); + it('should fail without mapping again when there is no test mapping to remove', async () => { getMappingsStub.resolves([listed(31002, 'Flux_App_web')]); From 4915696765750568286206642a4babc5975b87be Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:03:49 +0100 Subject: [PATCH 040/369] fix(upnp): verification's test mapping is not counted among the router's refusals routerRefusals counts the refusals of this node's own mappings that are the router's and not the port's: the evidence the availability check counts toward the router DOS. verifyUPNPsupport's test mapping is a probe it resolves itself - it clears room and maps once more, or runs on the api port the router already maps to it - so mapProtocol takes `counted` and verification passes false. A full table that stays full is counted from the node's own mappings, refused as they are refreshed. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 21 +++++++++++++-------- tests/unit/upnpService.test.js | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 04bb5ab0bb..dc374dcb62 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -115,9 +115,10 @@ const MAPPED = Object.freeze({ let routerRefusalCount = 0; /** - * How many mappings the router has refused since FluxOS started for a reason - * that is the router's own and not the port's: no answer, or any refusal but - * those PORT_REFUSALS names. Only ever goes up. + * How many of this node's mappings the router has refused since FluxOS started + * for a reason that is its own and not the port's: no answer, or any refusal + * but those PORT_REFUSALS names for a port it does not hold for something + * else. verifyUPNPsupport's test mapping is not counted. Only ever goes up. * @returns {number} */ function routerRefusals() { @@ -144,11 +145,15 @@ function mapsHere(mapping, port) { * @param {number} port * @param {string} protocol 'TCP' or 'UDP' * @param {string} description + * @param {object} [options] + * @param {boolean} [options.counted] whether a refusal of the router's own counts in + * routerRefusals; false for verifyUPNPsupport's test mapping, a probe it + * resolves itself * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} * `code` is the router's UPnP error code, null when it gave none; `routerWide` - * whether the refusal counts in routerRefusals. + * whether the refusal is the router's own rather than the port's. */ -async function mapProtocol(port, protocol, description) { +async function mapProtocol(port, protocol, description, { counted = true } = {}) { if (!Number.isInteger(port) || port < 1 || port > 65535) { return { mapped: false, code: null, reason: `${protocol} ${port} not mapped: not a port`, routerWide: false, @@ -177,7 +182,7 @@ async function mapProtocol(port, protocol, description) { const code = answered ? error.code : null; // no answer, with no code, is the router's too const routerWide = !PORT_REFUSALS.has(code) && !(heldElsewhere && code !== NO_PORT_MAPS_AVAILABLE); - if (routerWide) routerRefusalCount += 1; + if (routerWide && counted) routerRefusalCount += 1; return { mapped: false, code, reason: `${protocol} ${port} not mapped: ${error.message}`, routerWide, }; @@ -263,7 +268,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { // the remove would fail the verify, and a node with a custom api port or a // router IP exits on that. A test that fails leaves it held. mappedHere.add(+apiport + 3); - let testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test'); + let testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test', { counted: false }); // refused for a reason of the router's own, as a full table is: until UPnP is // verified the stale mapping cleanup never runs, so the test mappings earlier // runs left are removed here to make room. A router that did not answer is @@ -276,7 +281,7 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { } catch (error) { log.warn(`VerifyUPNPsupport - left-over test mappings not removed: ${error.message}`); } - if (removed > 0) testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test'); + if (removed > 0) testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test', { counted: false }); // A full table refuses every new mapping, so a node whose router already // maps its api port to it runs on what it holds: every mapping refused from // here on counts toward the router DOS, which takes the node out of service diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 00c38dff68..12259b689e 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -300,6 +300,24 @@ describe('upnpService tests', () => { sinon.assert.calledTwice(createMappingStub); }); + it('should count no refusal of its own test mapping toward the router\'s refusals', async () => { + const before = upnpService.routerRefusals(); + + expect(await upnpService.verifyUPNPsupport()).to.equal(true); + + sinon.assert.calledTwice(createMappingStub); + expect(upnpService.routerRefusals()).to.equal(before); + }); + + it('should count no refusal of its own test mapping when it fails, either', async () => { + getMappingsStub.resolves([]); + const before = upnpService.routerRefusals(); + + expect(await upnpService.verifyUPNPsupport()).to.equal(false); + + expect(upnpService.routerRefusals()).to.equal(before); + }); + [ [16127, 2], [16137, 3], From e07601d8b7747cd1d31725f4e931a98de7ba5c4c Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:23:39 +0100 Subject: [PATCH 041/369] fix(upnp): know this node's own mappings by their descriptions as the router stores them FRITZ!Box rewrites a mapping description's punctuation - 'Flux_App_web' comes back 'Flux.App.web' - so no description FluxOS reads from one ever matched those it makes: the stale mapping cleanup, verification's left-over test mapping cleanup and the fluxadm mapping check found nothing of this node's to remove. Every description comparison now goes through natUpnp.canonicalDescription, which every run of characters other than letters and digits makes one underscore. Needs @runonflux/nat-upnp with canonicalDescription. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 35 +++++++++++++++++++++----- tests/unit/upnpService.test.js | 39 +++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index dc374dcb62..0b8beaeed1 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -23,7 +23,9 @@ const FLUXADM_MAPPING_DESCRIPTION = 'Flux_Fluxadm_SSH'; // The descriptions this code gives the mappings it makes. Only these are ever // cleaned up: 'Flux_manual_entry' is the operator's own request through the // API, and anything else on the router - the owner's own forwards, another -// program's, a description the router truncated - is not ours to judge. +// program's, a description the router truncated - is not ours to judge. A +// description is compared as routers store it (natUpnp.canonicalDescription): +// FRITZ!Box keeps 'Flux_App_web' as 'Flux.App.web'. const OWNED_MAPPING_DESCRIPTIONS = new Set([ 'Flux_Backend_API', 'Flux_Backend_API_SSL', @@ -352,7 +354,7 @@ async function reconcileFluxadmMapping(apiport) { const mapping = await client.getMapping({ public: port, protocol: 'TCP' }); const ours = mapping !== null && mapping.local === true - && mapping.description === FLUXADM_MAPPING_DESCRIPTION; + && sameDescription(mapping.description, FLUXADM_MAPPING_DESCRIPTION); if (ours) { await client.removeMapping({ public: port, protocol: 'TCP' }); log.info(`fluxadm access - UPnP mapping for port ${port} removed`); @@ -536,9 +538,29 @@ async function removeMapUpnpPort(port, description = '') { */ function isOwnedMappingDescription(description) { if (typeof description !== 'string') return false; - if (description === FLUXADM_MAPPING_DESCRIPTION && fluxadmPort.isArcane) return false; - return OWNED_MAPPING_DESCRIPTIONS.has(description) - || OWNED_MAPPING_PREFIXES.some((prefix) => description.startsWith(prefix)); + if (sameDescription(description, FLUXADM_MAPPING_DESCRIPTION) && fluxadmPort.isArcane) return false; + return [...OWNED_MAPPING_DESCRIPTIONS].some((owned) => sameDescription(description, owned)) + || OWNED_MAPPING_PREFIXES.some((prefix) => descriptionStartsWith(description, prefix)); +} + +/** + * Whether two mapping descriptions are the same as a router stores them. + * @param {string} a + * @param {string} b + * @returns {boolean} + */ +function sameDescription(a, b) { + return natUpnp.canonicalDescription(a) === natUpnp.canonicalDescription(b); +} + +/** + * Whether a mapping description begins with a prefix, as a router stores them. + * @param {string} description + * @param {string} prefix + * @returns {boolean} + */ +function descriptionStartsWith(description, prefix) { + return natUpnp.canonicalDescription(description).startsWith(natUpnp.canonicalDescription(prefix)); } /** @@ -548,7 +570,8 @@ function isOwnedMappingDescription(description) { */ function isTestMappingDescription(description) { return typeof description === 'string' - && (TEST_MAPPING_DESCRIPTIONS.has(description) || description.startsWith(TEST_MAPPING_PREFIX)); + && ([...TEST_MAPPING_DESCRIPTIONS].some((test) => sameDescription(description, test)) + || descriptionStartsWith(description, TEST_MAPPING_PREFIX)); } /** diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 12259b689e..22ad2dd276 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -336,6 +336,19 @@ describe('upnpService tests', () => { }); }); + it('should know the test mappings earlier runs left as a FRITZ!Box stores their descriptions', async () => { + getMappingsStub.resolves([ + listed(31000, 'Flux.Test.App'), + listed(31001, 'Flux.Prelaunch.App.31001'), + listed(31002, 'Flux.App.web'), + ]); + + expect(await upnpService.verifyUPNPsupport()).to.equal(true); + + const removedPorts = removeMappingStub.getCalls().map((call) => call.args[0].public.port ?? call.args[0].public); + expect(removedPorts.filter((port) => port !== 16130)).to.deep.equal([31000, 31001]); + }); + it('should fail without mapping again when there is no test mapping to remove', async () => { getMappingsStub.resolves([listed(31002, 'Flux_App_web')]); @@ -595,6 +608,15 @@ describe('upnpService tests', () => { sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); }); + it('should know its own maintenance ssh mapping as a FRITZ!Box stores its description', async () => { + createMappingSpy.returns(true); + getMappingStub.resolves(fluxadmMapping({ description: 'Flux.Fluxadm.SSH' })); + + await runSetup(123); + + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); + }); + it('should remove nothing when the router holds no mapping of the maintenance ssh port', async () => { createMappingSpy.returns(true); @@ -1323,6 +1345,23 @@ describe('upnpService tests', () => { sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP', signal: sinon.match.instanceOf(AbortSignal) }); }); + it('should know its own mappings as a FRITZ!Box stores their descriptions, and nothing else', async () => { + // FRITZ!Box rewrites the punctuation: 'Flux_App_gone' comes back 'Flux.App.gone' + routerHas([ + mapping(31000, 'Flux.App.gone'), + mapping(31001, 'Flux.Prelaunch.App.31001'), + mapping(31002, 'Flux.Test.App'), + mapping(31003, 'Flux.manual.entry'), + mapping(31004, 'FluxOS.Reserved'), + ]); + + const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); + + expect(removed).to.equal(3); + const removedPorts = removeMappingStub.getCalls().map((call) => call.args[0].public.port); + expect(removedPorts).to.deep.equal([31000, 31001, 31002]); + }); + it('should remove nothing from an empty table', async () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); From 2a7ca83b8563494bc816b7c0f3a2c76c4a5a1e11 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:23:40 +0100 Subject: [PATCH 042/369] test(harness): 3001 verifies on a full table whose router stores descriptions as a FRITZ!Box does The stub gateway, given fritzDescriptions, stores each new mapping's description with its underscores and spaces as dots. 3001's new case starts the full table's left-over test mappings in that form. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 7 ++-- .../runner/framework/external-http-control.js | 6 ++- ...ehind-a-full-router-table-verifies-upnp.js | 42 +++++++++++++++++++ 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index af0f03250d..53186b9b56 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -789,7 +789,8 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { protocol: arg('NewProtocol').toUpperCase(), internalPort: Number(arg('NewInternalPort')), internalClient: arg('NewInternalClient'), - description: arg('NewPortMappingDescription'), + // a FRITZ!Box stores a description with its underscores and spaces as dots + description: gateway.fritzDescriptions ? arg('NewPortMappingDescription').replace(/[_ ]/g, '.') : arg('NewPortMappingDescription'), leaseDuration: Number(arg('NewLeaseDuration')), }); soapReply(res, action, {}); @@ -1018,14 +1019,14 @@ control.post('/storage-requests/reset', (req, res) => { // Put a node behind a UPnP gateway, with no mappings yet. control.post('/upnp-gateway', (req, res) => { const { - ip, externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, + ip, externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, fritzDescriptions = false, } = req.body || {}; if (!ip) { res.status(400).json({ error: 'ip is required' }); return; } state.upnpGateways[ip] = { - externalIp, capacity, mappings, refuseDeletes, requests: [], + externalIp, capacity, mappings, refuseDeletes, fritzDescriptions, requests: [], }; res.json({ ok: true }); }); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index 62acf74a0f..67305f0596 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -197,12 +197,14 @@ export async function resetStorageRequests() { * @param {object[]} [opts.mappings] - what its table holds already, as upnpGatewayState lists it * @param {number} [opts.refuseDeletes] - how many DeletePortMapping requests it refuses with * 501 before it removes anything; a refused request carries `refused: 501` in its state + * @param {boolean} [opts.fritzDescriptions] - store each new mapping's description as a + * FRITZ!Box does, its underscores and spaces as dots */ export async function enableUpnpGateway(ip, { - externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, + externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, fritzDescriptions = false, } = {}) { const res = await post('/upnp-gateway', { - ip, externalIp, capacity, mappings, refuseDeletes, + ip, externalIp, capacity, mappings, refuseDeletes, fritzDescriptions, }); if (!res.ok) throw new Error(`enableUpnpGateway: ${JSON.stringify(res)}`); } diff --git a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js index 83b9a9fca5..394909d586 100644 --- a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js +++ b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js @@ -81,6 +81,48 @@ describe('a node behind a full router table verifies UPnP', function () { }); }); +// The same full table on a router that stores descriptions as a FRITZ!Box does, its +// underscores and spaces as dots: the left-over test mappings read 'Flux.Test.App' and the +// like, and are this node's all the same. +describe('a node behind a full FRITZ!Box-style router table verifies UPnP', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + const fritz = (m) => ({ ...m, description: m.description.replace(/_/g, '.') }); + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); + await enableUpnpGateway(nodeIp, { + capacity: CAPACITY, mappings: [...LEFT_OVER.map(fritz), fritz(OTHERS)], fritzDescriptions: true, + }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('removes its left-over test mappings and maps its own ports in their room', async function () { + this.timeout(180000); + const corePorts = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2]; + let table = []; + await waitFor(async () => { + table = (await upnpGatewayState(nodeIp)).mappings; + return corePorts.every((port) => table.some((m) => m.externalPort === port)); + }, { timeout: 150000, interval: 2000, label: 'the node\'s own ports mapped' }).catch(() => {}); + + expect(table.map((m) => m.externalPort).sort(), 'the router\'s table').to.deep.equal([...corePorts, OTHERS.externalPort].sort()); + expect(table.find((m) => m.externalPort === API_PORT).description, 'canary: the router stores dots').to.equal('Flux.Backend.API'); + }); +}); + // The same router with nothing of this node's to remove: it holds the node's own four ports, // mapped by an earlier run, and other machines' mappings up to its capacity. The test mapping // finds no room, and the router already maps this node's api port to it, so the node runs on From c6a5bb8a76d6c67ba645bd7f389d89f2eabadd70 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:31:57 +0100 Subject: [PATCH 043/369] fix(upnp): the pre-install port test maps under the application's own description checkInstallingAppPortAvailable maps each port under Flux_App_, the description the installation maps it under, rather than Flux_Prelaunch_App_. A test mapping the router keeps - a removal it refused, an entry it will not rewrite - is then the application's as it installs, never an application's live port labelled as a test, which verification's left-over test mapping cleanup removes at the next start on a full table. Kept after an installation called off, it is an application mapping no installed application holds, for the stale mapping cleanup. The spawner passes the application's name. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appSpawner.js | 2 +- ZelBack/src/services/appNetwork/portManager.js | 16 ++++++++++++---- tests/unit/appSpawner.test.js | 13 ++++++++++++- tests/unit/portManager.test.js | 14 ++++++++++++-- 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appSpawner.js b/ZelBack/src/services/appLifecycle/appSpawner.js index d3cd50f53b..faf8626eda 100644 --- a/ZelBack/src/services/appLifecycle/appSpawner.js +++ b/ZelBack/src/services/appLifecycle/appSpawner.js @@ -700,7 +700,7 @@ async function trySpawningGlobalApplication() { // Note: User-blocked port check happens earlier (line ~353) before Docker Hub calls // Check if ports are publicly available - critical for proper Flux network operation - const portVerdict = await portManager.checkInstallingAppPortAvailable(appPorts); + const portVerdict = await portManager.checkInstallingAppPortAvailable(appPorts, appSpecifications.name); if (portVerdict.ok === false) { log.error(`trySpawningGlobalApplication - Some of application ports of ${appSpecifications.name} are not available publicly. Installation aborted.`); // The cause lives in portManager, which says which port and which peers; diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 42fe960f1a..235573cde7 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -817,11 +817,19 @@ function portNotOurs(portsToTest, answered, token) { * about the network, and only one of them is evidence. The object returned is * the one published to the harness bus, so the two cannot drift. * + * Behind UPnP each port is mapped for the test under the application's own + * description, the one its installation maps it under: a mapping the router + * keeps after the test - a removal it refused, an entry it will not rewrite - + * is then the application's as it installs, or a stale one of an application + * this node does not hold, for the cleanup. + * * @param {Array} portsToTest Array of ports to test + * @param {string} appName the application the ports are for * @returns {Promise<{ok: boolean, reason: string, port?: number, peers?: string[], * readings?: object, asked?: string[], silent?: boolean}>} */ -async function checkInstallingAppPortAvailable(portsToTest = []) { +async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { + const description = `Flux_App_${appName}`; const beforeAppInstallTestingServers = []; // One secret for this whole test run, published by our own test servers and // never sent to the peer. The peer returns what it read; we compare. @@ -873,7 +881,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { } if (isUPNP) { // eslint-disable-next-line no-await-in-loop - const upnpMapResult = await upnpService.mapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`); + const upnpMapResult = await upnpService.mapUpnpPort(portToTest, description); if (!upnpMapResult.mapped) { throw new Error(`Failed to create map UPNP port: ${upnpMapResult.reason}`); } @@ -1131,7 +1139,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { } if (isUPNP) { // eslint-disable-next-line no-await-in-loop - await upnpService.removeMapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`); + await upnpService.removeMapUpnpPort(portToTest, description); } } // Close all test servers and wait for them to finish @@ -1158,7 +1166,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { } if (isUPNP) { // eslint-disable-next-line no-await-in-loop - await upnpService.removeMapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`).catch((e) => log.error(e)); + await upnpService.removeMapUpnpPort(portToTest, description).catch((e) => log.error(e)); } } // Close all test servers and wait for them to finish diff --git a/tests/unit/appSpawner.test.js b/tests/unit/appSpawner.test.js index 0365fe4b5c..0d1866269c 100644 --- a/tests/unit/appSpawner.test.js +++ b/tests/unit/appSpawner.test.js @@ -236,7 +236,7 @@ describe('appSpawner tests', () => { '../appNetwork/portManager': { ensureApplicationPortsNotUsed: sinon.stub().resolves(), siblingHoldingPort: sinon.stub().resolves(opts.siblingHoldingPort ?? null), - checkInstallingAppPortAvailable: sinon.stub().resolves({ + checkInstallingAppPortAvailable: (opts.portCheck ?? sinon.stub()).resolves({ ok: opts.portsAvailable ?? true, reason: (opts.portsAvailable ?? true) ? 'proven' : 'notOurs', }), @@ -1045,6 +1045,17 @@ describe('appSpawner tests', () => { sinon.assert.notCalled(installStub); }); + it('should test the ports for the application it is spawning, by name', async () => { + const portCheck = sinon.stub(); + buildModule({ + aggregateResult: [spawnableApp], appSpec: fullSpec, errorCount: 0, portsAvailable: false, portCheck, + }); + + await appSpawner.trySpawningGlobalApplication(); + + sinon.assert.calledOnceWithExactly(portCheck, sinon.match.array, fullSpec.name); + }); + it('should add to short-term cache when network error count >= 5', async () => { buildModule({ aggregateResult: [spawnableApp], appSpec: fullSpec, errorCount: 5 }); await appSpawner.trySpawningGlobalApplication().catch(() => {}); diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index a976c56722..d686e07342 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -1261,7 +1261,7 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', return answer; }); - return portManager.checkInstallingAppPortAvailable([port]); + return portManager.checkInstallingAppPortAvailable([port], 'web'); }; // The bug: a verdict and "nobody has decided" were the same value, so a last @@ -1376,6 +1376,16 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(result).to.deep.equal({ ok: false, reason: 'error' }); sinon.assert.notCalled(axios.post); - sinon.assert.calledWith(removeMapping, port, `Flux_Prelaunch_App_${port}`); + sinon.assert.calledWith(removeMapping, port, 'Flux_App_web'); + }); + + it('maps the port under test under the application\'s own description, the one it installs under', async () => { + upnpService.isUPNP.returns(true); + const mapUpnpPort = sinon.stub(upnpService, 'mapUpnpPort').resolves(REFUSED); + sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); + + await withPeers([UNREACHABLE]); + + sinon.assert.calledOnceWithExactly(mapUpnpPort, port, 'Flux_App_web'); }); }); From 7527522cff9fc13ca54be10dc29cd10a6a7473cf Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:19:27 +0100 Subject: [PATCH 044/369] chore(upnp): nat-upnp ~2.0.4 2.0.2 resolves the node's LAN address on every request, so a node whose address changes maps at its new one. 2.0.3 keeps a router's connection when the router answers keep-alive, which lets the TP-Link TPOS and VxWorks routers' tables be listed, and ends a walk at the router's fault whatever its code. 2.0.4 sends a discovery search again until a router answers, takes only an IPv4 router, applies the kept-connection idle limit only in the pool, and adds canonicalDescription. Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 69b2b4a90f..6dd6c00528 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "@aws-sdk/client-ecr": "^3.922.0", "@azure/identity": "^4.13.0", "@isaacs/ttlcache": "~1.4.1", - "@runonflux/nat-upnp": "~2.0.1", + "@runonflux/nat-upnp": "~2.0.4", "apicache": "~1.6.3", "archiver": "~7.0.1", "axios": "~1.13.6", From 5e3108fb354b7cd1ccf734d3e4bb56fb8e0a3dbc Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 18:22:13 +0100 Subject: [PATCH 045/369] chore(apps): a cache hit for a decrypted enterprise spec is not logged Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appQuery/appQueryService.js | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index 9497f5ee5d..da109e3918 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -48,10 +48,7 @@ async function decryptEnterpriseSpec(spec) { const cache = fluxCaching.default.enterpriseAppDecryptionCache; const cached = cache.get(cacheKey); - if (cached) { - log.info(`Using cached decrypted app for ${spec.name} (${cacheKey})`); - return cached; - } + if (cached) return cached; const failure = decryptFailures.get(cacheKey); if (failure && Date.now() - failure.at < DECRYPT_FAILURE_WINDOW_MS) { From b682ec0fb0a90803e1ac653c0f2b3249ab4b0d29 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 10:30:36 +0100 Subject: [PATCH 046/369] fix(db): a failed database call no longer exits FluxOS or leaves a flag set Work started from a timer or a listener has nothing awaiting it, so a database call that fails there is caught and logged where it starts: - the block explorer's request for a block's app payments, and the payment record read it starts with: a record that cannot be read leaves the hash unresolved, as a missing one does - removing every app when the node loses confirmation or the daemon goes stale - the explorer reindex and rescan requests: a failed drop or height write is answered once and releases the operation; a rescan without a height is answered once Flags that a failure left set: - the daemon health monitor marks its removal done only once the installed apps were read; getInstalledApps rejects on a failed read instead of answering an empty list - the moves this node recorded before a restart count as loaded only once they were read, and the node keeps listening for peers until they have been - a block processing stop that does not take leaves processing allowed to continue Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 20 +-- .../services/appMessaging/messageVerifier.js | 11 +- .../appMonitoring/daemonHealthMonitor.js | 4 +- .../appMonitoring/nodeStatusMonitor.js | 20 ++- ZelBack/src/services/explorerService.js | 124 +++++++++++------- ZelBack/src/services/fluxCommunication.js | 15 ++- tests/unit/daemonHealthMonitor.test.js | 17 +++ tests/unit/explorerService.test.js | 84 ++++++++++++ tests/unit/fluxCommunication.test.js | 12 ++ tests/unit/messageVerifier.test.js | 10 ++ tests/unit/nodeStatusMonitor.test.js | 48 +++++++ tests/unit/registryManager.test.js | 8 +- 12 files changed, 297 insertions(+), 76 deletions(-) diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index b1ef43256a..e2a7b41613 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -1648,22 +1648,14 @@ async function getAllAppsInformation() { /** * Get installed apps information - * @returns {Promise} Array of installed apps + * @returns {Promise} Array of installed apps. Rejects when the read fails: + * an unread list is not an empty one. */ async function getInstalledApps() { - try { - const localDb = dbHelper.databaseConnection(); - const localDatabase = localDb.db(config.database.appslocal.database); - - const query = {}; - const projection = { projection: { _id: 0 } }; - - const installedApps = await dbHelper.findInDatabase(localDatabase, localAppsInformation, query, projection); - return installedApps; - } catch (error) { - log.error(`Error getting installed apps: ${error.message}`); - return []; - } + const localDb = dbHelper.databaseConnection(); + const localDatabase = localDb.db(config.database.appslocal.database); + const projection = { projection: { _id: 0 } }; + return dbHelper.findInDatabase(localDatabase, localAppsInformation, {}, projection); } /** diff --git a/ZelBack/src/services/appMessaging/messageVerifier.js b/ZelBack/src/services/appMessaging/messageVerifier.js index 14825d65c6..37f6fcb2ac 100644 --- a/ZelBack/src/services/appMessaging/messageVerifier.js +++ b/ZelBack/src/services/appMessaging/messageVerifier.js @@ -753,10 +753,17 @@ async function paymentRecordOf(hash) { * @param {number} height * @param {number} valueSat * @param {number} [i] attempt - * @returns {Promise} whether the message is stored + * @returns {Promise} whether the message is stored. False when the payment record + * could not be read: the hash stays unresolved and is asked about again. */ async function checkAndRequestApp(hash, txid, height, valueSat, i = 0) { - const record = await paymentRecordOf(hash); + let record; + try { + record = await paymentRecordOf(hash); + } catch (error) { + log.error(`checkAndRequestApp - payment record for ${hash} could not be read, left unresolved: ${error.message}`); + return false; + } if (!record) { log.warn(`checkAndRequestApp - no payment recorded for ${hash} (requested from txid ${txid}), not stored`); return false; diff --git a/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js b/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js index fde0e7f1e6..df1e9a5f40 100644 --- a/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js +++ b/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js @@ -19,10 +19,10 @@ const REMOVAL_DELAY = 3 * 60 * 1000; // 3 minutes between app removals */ async function removeAllApps(reason) { try { - allAppsRemoved = true; // Set flag to prevent repeated attempts - // Get installed apps (follows pattern from checkAndRemoveEnterpriseAppsOnNonArcane) const installedApps = await registryManager.getInstalledApps(); + // Set once the list is read, so a failed read is retried on the next check. + allAppsRemoved = true; if (!installedApps || installedApps.length === 0) { log.info('No apps installed, nothing to remove'); diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 49c6bebdb0..a08fe4b1c5 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -47,16 +47,32 @@ async function removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason) } } +/** + * Removal started from a listener: nothing awaits it, so its failure is logged + * here. The periodic monitor tries again on its next pass. + * @param {Function} installedAppsFn + * @param {Function} removeAppLocallyFn + * @param {string} reason + * @returns {Promise} + */ +async function removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, reason) { + try { + await removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason); + } catch (error) { + log.error(`monitorNodeStatus - app removal (${reason}) failed: ${error.message}`); + } +} + function initialize(installedAppsFn, removeAppLocallyFn) { nodeConfirmationService.onConfirmationChange((confirmed) => { if (!confirmed) { log.info('monitorNodeStatus - Confirmation lost, triggering immediate app removal'); - removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, 'node lost confirmation'); + removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, 'node lost confirmation'); } }); nodeConfirmationService.onDaemonStale(() => { log.info('monitorNodeStatus - Daemon stale, triggering app removal'); - removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, 'daemon unreachable'); + removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, 'daemon unreachable'); }); } diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index be74343001..eb1fad853b 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -436,35 +436,48 @@ async function insertTransactions(transactions, database) { * @param {array} apps array with appstransactions to be processed. * @param {string} database Database. */ +/** + * Stores or requests the messages of a block's app payments. Runs from a timer + * nothing awaits, so it logs its own failure: the hashes it leaves unresolved + * are found again by hash sync. + * @param {array} apps the block's app payments. + * @returns {Promise} + */ +async function requestAppHashes(apps) { + try { + const appsToRemove = []; + let hasUnresolved = false; + // eslint-disable-next-line no-restricted-syntax + for (const app of apps) { + // eslint-disable-next-line no-await-in-loop + const messageReceived = await messageVerifier.checkAndRequestApp(app.hash, app.txid, app.height, app.value, 2); + if (messageReceived) { + appsToRemove.push(app); + } else { + hasUnresolved = true; + } + } + if (hasUnresolved) { + appSyncEvents.emit(SYNC_EVENTS.HASH_UNRESOLVED); + } + const remaining = apps.filter((item) => !appsToRemove.includes(item)); + while (remaining.length > 500) { + messageVerifier.checkAndRequestMultipleApps(remaining.splice(0, 500)); + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(30 * 1000); // delay 30 seconds + } + if (remaining.length > 0) { + messageVerifier.checkAndRequestMultipleApps(remaining); + } + } catch (error) { + log.error(`Explorer - requestAppHashes - ${apps.length} app payments: ${error.message}`); + } +} + async function insertAndRequestAppHashes(apps, database) { if (apps.length > 0) { await insertTransactions(apps, database); - setTimeout(async () => { - const appsToRemove = []; - let hasUnresolved = false; - // eslint-disable-next-line no-restricted-syntax - for (const app of apps) { - // eslint-disable-next-line no-await-in-loop - const messageReceived = await messageVerifier.checkAndRequestApp(app.hash, app.txid, app.height, app.value, 2); - if (messageReceived) { - appsToRemove.push(app); - } else { - hasUnresolved = true; - } - } - if (hasUnresolved) { - appSyncEvents.emit(SYNC_EVENTS.HASH_UNRESOLVED); - } - const remaining = apps.filter((item) => !appsToRemove.includes(item)); - while (remaining.length > 500) { - messageVerifier.checkAndRequestMultipleApps(remaining.splice(0, 500)); - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(30 * 1000); // delay 30 seconds - } - if (remaining.length > 0) { - messageVerifier.checkAndRequestMultipleApps(remaining); - } - }, 1); + setTimeout(requestAppHashes, 1, apps); } } /** @@ -1676,6 +1689,8 @@ async function checkBlockProcessingStopped(i, callback) { if (j < 12) { checkBlockProcessingStopped(j, callback); } else { + // Not stopped: the block in flight keeps chaining to the next one. + blockProccessingCanContinue = true; const errMessage = messageHelper.createErrorMessage('Unknown error occured. Try again later.'); callback(errMessage); } @@ -1764,25 +1779,23 @@ async function reindexExplorer(req, res) { res.json(errMessage); } else { operationBlocked = true; - const dbopen = dbHelper.databaseConnection(); - const database = dbopen.db(config.database.daemon.database); - const resultOfDropping = await dbHelper.dropCollection(database, scannedHeightCollection).catch((error) => { - if (error.message !== 'ns not found') { - operationBlocked = false; - log.error(error); - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - res.json(errMessage); - } - }); - operationBlocked = false; - if (resultOfDropping === true || resultOfDropping === undefined) { - initiateBlockProcessor(true, false); - const message = messageHelper.createSuccessMessage('Explorer database reindex initiated'); - res.json(message); - } else { - const errMessage = messageHelper.createErrorMessage(resultOfDropping, 'Collection dropping error'); - res.json(errMessage); + try { + const dbopen = dbHelper.databaseConnection(); + const database = dbopen.db(config.database.daemon.database); + await dbHelper.dropCollection(database, scannedHeightCollection).catch((error) => { + // A collection that is already gone is the reindex's starting point. + if (error.message !== 'ns not found') throw error; + }); + } catch (error) { + operationBlocked = false; + log.error(error); + res.json(messageHelper.createErrorMessage(error.message, error.name, error.code)); + return; } + operationBlocked = false; + initiateBlockProcessor(true, false); + const message = messageHelper.createSuccessMessage('Explorer database reindex initiated'); + res.json(message); } }); } else { @@ -1813,6 +1826,7 @@ async function rescanExplorer(req, res) { if (!blockheight) { const errMessage = messageHelper.createErrorMessage('No blockheight provided'); res.json(errMessage); + return; } blockheight = serviceHelper.ensureNumber(blockheight); const dbopen = dbHelper.databaseConnection(); @@ -1839,17 +1853,23 @@ async function rescanExplorer(req, res) { checkBlockProcessingStopped(i, async (response) => { if (response.status === 'error') { res.json(response); + } else if (operationBlocked) { + res.json(messageHelper.createErrorMessage('Operation blocked')); } else { - if (operationBlocked) { - throw new Error('Operation blocked'); - } operationBlocked = true; const update = { $set: { generalScannedHeight: blockheight } }; const options = { upsert: true, }; - // update scanned Height in scannedBlockHeightCollection - await dbHelper.updateOneInDatabase(database, scannedHeightCollection, query, update, options); + try { + // update scanned Height in scannedBlockHeightCollection + await dbHelper.updateOneInDatabase(database, scannedHeightCollection, query, update, options); + } catch (error) { + operationBlocked = false; + log.error(error); + res.json(messageHelper.createErrorMessage(error.message, error.name, error.code)); + return; + } operationBlocked = false; initiateBlockProcessor(true, false); const message = messageHelper.createSuccessMessage(`Explorer rescan from blockheight ${blockheight} initiated`); @@ -1938,6 +1958,11 @@ function setBlockProccessingCanContinue(value) { blockProccessingCanContinue = value; } +// testing purposes +function getBlockProccessingCanContinue() { + return blockProccessingCanContinue; +} + // testing purposes function setIsInInitiationOfBP(value) { isInInitiationOfBP = value; @@ -1980,6 +2005,7 @@ module.exports = { processTransaction, processStandard, setBlockProccessingCanContinue, + getBlockProccessingCanContinue, setIsInInitiationOfBP, setZelAppSpecsMigrationDone, restoreDatabaseToBlockheightState, diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 9c6bd5b0c1..305fd43fcd 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -555,7 +555,6 @@ function dropExpiredOwnMoves() { * key signed is the previous holder's. */ async function loadRecordedOwnMoves() { - recordedLoaded = true; try { const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); const ownKey = await fluxNetworkHelper.getFluxNodePublicKey(); @@ -565,6 +564,8 @@ async function loadRecordedOwnMoves() { for (const [oldIp, { to, at, signer }] of recorded) { if (signer === ownKey) addPendingOwnMove(oldIp, to, at); } + // Only a completed read counts: until then each connecting peer asks again. + recordedLoaded = true; } catch (error) { log.warn(`loadRecordedOwnMoves - this node's recorded moves cannot be read: ${error.message}`); } @@ -591,7 +592,7 @@ async function sendOwnMovesTo(peerKey) { log.error(`sendOwnMovesTo - the move from ${move.oldIp} to ${move.newIp} could not be sent to ${peerKey}: ${error.message}`); } } - if (!pendingOwnMoves.length) stopSendingOwnMoves(); + stopSendingOwnMovesWhenDone(); } function startSendingOwnMoves() { @@ -606,6 +607,14 @@ function stopSendingOwnMoves() { peerManager.off('peerConnected', sendOwnMovesTo); } +/** + * Stop listening for peers once nothing is left to send: every pending move is + * gone and the recorded ones have been read. + */ +function stopSendingOwnMovesWhenDone() { + if (recordedLoaded && !pendingOwnMoves.length) stopSendingOwnMoves(); +} + /** * A peer announced, or relayed, a message this node holds the hash of. When it * is one of this node's own moves, the move has gone further than the peers @@ -619,7 +628,7 @@ function noteOwnMoveHeardBack(messageHash) { if (index === -1) return false; const [move] = pendingOwnMoves.splice(index, 1); log.info(`Address move from ${move.oldIp} to ${move.newIp} heard back from the network - no longer sent`); - if (!pendingOwnMoves.length) stopSendingOwnMoves(); + stopSendingOwnMovesWhenDone(); return true; } diff --git a/tests/unit/daemonHealthMonitor.test.js b/tests/unit/daemonHealthMonitor.test.js index b291ac7ac8..65c63e840d 100644 --- a/tests/unit/daemonHealthMonitor.test.js +++ b/tests/unit/daemonHealthMonitor.test.js @@ -300,6 +300,23 @@ describe('daemonHealthMonitor tests', () => { expect(logStub.error.calledWith('Error in daemon health check: RPC error')).to.be.true; }); + it('should try the removal again when the installed apps could not be read', async () => { + clock = sinon.useFakeTimers(); + + registryManagerStub.getInstalledApps.rejects(new Error('connection timed out')); + + daemonServiceMiscRpcsStub.isDaemonSynced.returns({ data: { synced: false } }); + await daemonHealthMonitor.checkDaemonHealthAndCleanup(); + clock.tick(2 * 60 * 60 * 1000); + await daemonHealthMonitor.checkDaemonHealthAndCleanup(); + expect(appUninstallerStub.removeAppLocally.called).to.be.false; + + registryManagerStub.getInstalledApps.resolves([{ name: 'app1' }]); + await daemonHealthMonitor.checkDaemonHealthAndCleanup(); + + expect(appUninstallerStub.removeAppLocally.callCount).to.equal(1); + }); + it('should reset allAppsRemoved flag when daemon recovers', async () => { clock = sinon.useFakeTimers(); diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 482e03d41d..bf6faef7f3 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -11,6 +11,7 @@ const daemonServiceAddressRpcs = require('../../ZelBack/src/services/daemonServi const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/daemonServiceMiscRpcs'); const daemonServiceUtils = require('../../ZelBack/src/services/daemonService/daemonServiceUtils'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); const globalState = require('../../ZelBack/src/services/utils/globalState'); const log = require('../../ZelBack/src/lib/log'); const { requireMongo } = require('./dbTestHelper'); @@ -2676,4 +2677,87 @@ describe('explorerService tests', () => { sinon.assert.notCalled(updateOneStub); }); }); + + describe('explorer reset requests', () => { + let res; + let clock; + + beforeEach(() => { + res = { json: sinon.spy(), status: sinon.stub().returnsThis() }; + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: sinon.fake.returns({}) }); + sinon.stub(log, 'error'); + explorerService.setIsInInitiationOfBP(false); + explorerService.setBlockProccessingCanContinue(true); + }); + + afterEach(() => { + clock?.restore(); + clock = undefined; + explorerService.setIsInInitiationOfBP(false); + explorerService.setBlockProccessingCanContinue(true); + sinon.restore(); + }); + + async function settle() { + for (let k = 0; k < 5; k += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } + } + + it('reindex answers once with the error and releases the operation when the drop fails', async () => { + const dropCollection = sinon.stub(dbHelper, 'dropCollection').rejects(new Error('connection timed out')); + + await explorerService.reindexExplorer({ params: {} }, res); + await settle(); + + sinon.assert.calledOnce(res.json); + sinon.assert.calledWithMatch(res.json, { status: 'error', data: { message: 'connection timed out' } }); + + dropCollection.rejects(new Error('connection timed out again')); + res.json.resetHistory(); + await explorerService.reindexExplorer({ params: {} }, res); + await settle(); + sinon.assert.calledWithMatch(res.json, { data: { message: 'connection timed out again' } }); + }); + + it('rescan answers once with the error and releases the operation when the height write fails', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves({ generalScannedHeight: 2000 }); + const updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new Error('connection timed out')); + + await explorerService.rescanExplorer({ params: { blockheight: 1000 } }, res); + await settle(); + + sinon.assert.calledOnce(res.json); + sinon.assert.calledWithMatch(res.json, { status: 'error', data: { message: 'connection timed out' } }); + + updateOne.rejects(new Error('connection timed out again')); + res.json.resetHistory(); + await explorerService.rescanExplorer({ params: { blockheight: 1000 } }, res); + await settle(); + sinon.assert.calledWithMatch(res.json, { data: { message: 'connection timed out again' } }); + }); + + it('rescan without a block height answers once', async () => { + const findOne = sinon.stub(dbHelper, 'findOneInDatabase'); + + await explorerService.rescanExplorer({ params: {} }, res); + + sinon.assert.calledOnce(res.json); + sinon.assert.calledWithMatch(res.json, { status: 'error', data: { message: 'No blockheight provided' } }); + sinon.assert.notCalled(findOne); + }); + + it('a stop that does not take leaves block processing allowed to continue', async () => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + explorerService.setIsInInitiationOfBP(true); + + await explorerService.stopBlockProcessing({}, res); + await clock.tickAsync(13000); + + sinon.assert.calledWithMatch(res.json, { status: 'error' }); + expect(explorerService.getBlockProccessingCanContinue()).to.equal(true); + }); + }); }); diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index fd786e3364..6d4f592a36 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -2835,6 +2835,18 @@ describe('fluxCommunication tests', () => { heardBack('10.0.0.1:16127'); }); + it('at startup, reads the recorded moves again on the next connection when the read fails', async () => { + registryManager.addressMovesTo.withArgs(NEW).onFirstCall().rejects(new Error('connection timed out')); + registryManager.addressMovesTo.withArgs(NEW).onSecondCall().resolves(new Map([[OLD, { to: NEW, at: Date.now() - 60000, signer: '04own' }]])); + + fluxCommunication.announceRecordedOwnMovesWhenConnected(); + + expect(await connect('10.0.0.1:16127')).to.deep.equal([]); + expect(peerManager.listeners('peerConnected').some((listener) => listener.name === 'sendOwnMovesTo')).to.equal(true); + expect(await connect('10.0.0.2:16127')).to.deep.equal([`${OLD}>${NEW}`]); + heardBack('10.0.0.2:16127'); + }); + it('at startup with nothing recorded, stops listening after the first connection', async () => { fluxCommunication.announceRecordedOwnMovesWhenConnected(); diff --git a/tests/unit/messageVerifier.test.js b/tests/unit/messageVerifier.test.js index 21f3d05d3d..baff1f77f0 100644 --- a/tests/unit/messageVerifier.test.js +++ b/tests/unit/messageVerifier.test.js @@ -330,6 +330,16 @@ describe('messageVerifier tests', () => { expect(result).to.equal(false); expect(storeAppPermanentMessageStub.called).to.equal(false); }); + + it('should leave the hash unresolved when its payment record cannot be read', async () => { + const findOneInDatabase = sinon.stub().rejects(new Error('connection timed out')); + const verifier = buildVerifier({ dbHelperOverrides: { findOneInDatabase } }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(result).to.equal(false); + expect(storeAppPermanentMessageStub.called).to.equal(false); + }); }); describe('an update that has expired by itself', () => { diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index e6f8f6ad04..9166d325f8 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -66,4 +66,52 @@ describe('nodeStatusMonitor tests', () => { // Note: This is a recursive monitoring function with complex business logic // Full testing would require integration tests or extensive mocking }); + + describe('initialize', () => { + let listeners; + let monitor; + + beforeEach(() => { + listeners = {}; + monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { + '../dbHelper': {}, + '../serviceHelper': { delay: sinon.stub().resolves() }, + '../fluxNetworkHelper': {}, + '../fluxCommunicationUtils': {}, + '../appMessaging/messageStore': {}, + '../networkStateService': {}, + '../nodeConfirmationService': { + onConfirmationChange: (cb) => { listeners.confirmation = cb; }, + onDaemonStale: (cb) => { listeners.daemonStale = cb; }, + canSendMessages: () => true, + }, + '../../lib/log': logStub, + }); + }); + + // Nothing awaits a listener's removal: a read that fails there is logged, + // never left to escape as an unhandled rejection that exits FluxOS. + it('logs a removal whose installed-apps read fails when confirmation is lost', async () => { + const installedAppsFn = sinon.stub().rejects(new Error('connection timed out')); + const removeAppLocallyFn = sinon.stub(); + monitor.initialize(installedAppsFn, removeAppLocallyFn); + + listeners.confirmation(false); + await new Promise((resolve) => { setImmediate(resolve); }); + + sinon.assert.calledOnce(installedAppsFn); + sinon.assert.notCalled(removeAppLocallyFn); + sinon.assert.calledWithMatch(logStub.error, /node lost confirmation\) failed: connection timed out/); + }); + + it('logs a removal whose installed-apps read fails when the daemon goes stale', async () => { + const installedAppsFn = sinon.stub().resolves({ status: 'error' }); + monitor.initialize(installedAppsFn, sinon.stub()); + + listeners.daemonStale(); + await new Promise((resolve) => { setImmediate(resolve); }); + + sinon.assert.calledWithMatch(logStub.error, /daemon unreachable\) failed/); + }); + }); }); diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 1b9773af46..9878c2a6e0 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1629,13 +1629,13 @@ describe('registryManager tests', () => { expect(result).to.be.an('array'); }); - it('should handle errors and return empty array', async () => { + it('should reject when the read fails, never answer an empty list', async () => { sinon.stub(dbHelper, 'findInDatabase').rejects(new Error('Database error')); - const result = await registryManager.getInstalledApps(); + const error = await registryManager.getInstalledApps().catch((err) => err); - expect(result).to.be.an('array'); - expect(result).to.be.empty; + expect(error).to.be.instanceOf(Error); + expect(error.message).to.equal('Database error'); }); }); From 0935f8430f9110d69f4694c7eda5c87fa42400ae Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 10:46:11 +0100 Subject: [PATCH 047/369] fix(db): an unread runtime record holds a component, and an uninstall finishes its cleanup A component's runtime record carries its operator stop lock and its restart ladders. getState rejects when the read fails, instead of answering no record: - a reconcile pass whose stop lock or restart generation cannot be read starts nothing and is retried; the generation is read before the start, so a restart requested while the start runs is still owed after it - the primary election leaves out, for the pass, a component whose stop lock cannot be read - a restart is not recorded over a ladder that could not be read An uninstall's last steps, after the containers and volume are gone, are the runtime record and the installed-app row. A step that fails is retried until it succeeds, and the app is held as departing meanwhile: the reconciler leaves a departing app alone, so a row that outlived its containers is never rebuilt on an empty volume, and a stop lock is never left behind for a reinstall. appsRuntimeState.remove rejects on a failed delete. Merging duplicate runtime records writes the merge onto the newest twin before removing the others, so a failure part way leaves duplicates rather than none. isOperatorStopped had no caller and is removed. The test fixture that resets globalState leaves an object's functions in place. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 12 ++- .../services/appLifecycle/appUninstaller.js | 43 ++++++++- .../services/appManagement/appController.js | 2 +- .../appManagement/appsRuntimeState.js | 68 +++++--------- .../services/appMonitoring/appReconciler.js | 24 +++-- tests/unit/advancedWorkflows.test.js | 20 ++-- tests/unit/appReconciler.test.js | 40 ++++++-- tests/unit/appUninstaller.test.js | 40 +++++++- tests/unit/appsRuntimeState.test.js | 93 +++++++++++++++++-- tests/unit/fixtures/globalState.js | 3 +- 10 files changed, 256 insertions(+), 89 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 6f3de20123..81b9f307cb 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -5133,8 +5133,16 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // runs all of them on one node, and every other holder runs none. Checked // for all g: mode apps, not just those in cache with restarted flag - the // cache tracks sync state, but shouldn't gate primary selection. - // eslint-disable-next-line no-await-in-loop - const members = await electedMembers(installedApp); + let members; + try { + // eslint-disable-next-line no-await-in-loop + members = await electedMembers(installedApp); + } catch (error) { + // An unread stop lock is not a lifted one: not a candidate this pass. + log.warn(`masterSlaveApps: ${installedApp.name} stop locks could not be read - excluded from primary election this pass: ${error.message}`); + // eslint-disable-next-line no-continue + continue; + } if (members.length) { // The anchor keys this app's election records and names it in the counters. const { identifier } = members[0]; diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 53c0635724..23d566f793 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -57,6 +57,39 @@ function setOnComponentRemoved(callback) { onComponentRemoved = callback; } +// Waits between attempts at a cleanup step of an uninstall that failed, the +// last repeated until the step succeeds. +const CLEANUP_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; + +/** + * Runs a cleanup step of an uninstall whose containers are already gone, and + * retries it until it succeeds. While a step is outstanding the app is held as + * departing: the reconciler leaves it alone, so a row that outlived its + * containers is never rebuilt on an empty volume, and announcements leave it out. + * @param {string} appName The app the step cleans up after. + * @param {string} what What the step removes, for the log. + * @param {function(): Promise} step + * @param {number} [attempt] Attempts already made. + * @returns {Promise} Whether the step succeeded on this attempt. + */ +async function runCleanupStep(appName, what, step, attempt = 0) { + try { + await step(); + if (attempt > 0) { + globalState.departingApps.leave(appName); + log.info(`${appName}: ${what} removed after ${attempt} failed attempt(s)`); + } + return true; + } catch (error) { + if (attempt === 0) globalState.departingApps.enter(appName); + const delay = CLEANUP_RETRY_MS[Math.min(attempt, CLEANUP_RETRY_MS.length - 1)]; + log.error(`${appName}: ${what} could not be removed (${error.message}); held as departing, retrying in ${Math.round(delay / 1000)}s`); + const timer = setTimeout(runCleanupStep, delay, appName, what, step, attempt + 1); + if (timer.unref) timer.unref(); + return false; + } +} + /** * Stop Syncthing app and clean up cache * @param {string} monitoredName - Monitored app name @@ -982,7 +1015,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // eslint-disable-next-line no-restricted-syntax for (const identifier of removedIdentifiers) { // eslint-disable-next-line no-await-in-loop - await appsRuntimeState.remove(identifier); + await runCleanupStep(appName, `runtime state of ${identifier}`, () => appsRuntimeState.remove(identifier)); if (onComponentRemoved) onComponentRemoved(identifier); } @@ -1097,7 +1130,11 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen res.write(serviceHelper.ensureString(databaseStatus)); if (res.flush) res.flush(); } - await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); + const rowDeleted = await runCleanupStep( + appName, + 'installed-app row', + () => dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection), + ); // The app is gone for good: nothing reconciles an app with no row, so its // removal records have no reader left. Only this full-uninstall path clears // them - softRemoveAppLocally deletes the row too, but as one step of a @@ -1106,7 +1143,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // tampering. dockerService.clearFluxRemovedContainers(appName); const databaseStatus2 = { - status: 'Database cleaned', + status: rowDeleted ? 'Database cleaned' : 'Database cleanup pending, retrying', }; log.info(databaseStatus2); if (res) { diff --git a/ZelBack/src/services/appManagement/appController.js b/ZelBack/src/services/appManagement/appController.js index 2f9a46c7bc..44e3cfefce 100644 --- a/ZelBack/src/services/appManagement/appController.js +++ b/ZelBack/src/services/appManagement/appController.js @@ -392,7 +392,7 @@ async function containersRestarted(ids) { // eslint-disable-next-line no-restricted-syntax for (const id of ids) { // eslint-disable-next-line no-await-in-loop - const state = await appsRuntimeState.getState(id); + const state = await appsRuntimeState.getState(id).catch(() => null); if (!state) return { settled: false, reason: 'its state could not be read' }; if ((state.actuatedRestartGeneration ?? 0) < (state.restartGeneration ?? 0)) { // eslint-disable-next-line no-await-in-loop diff --git a/ZelBack/src/services/appManagement/appsRuntimeState.js b/ZelBack/src/services/appManagement/appsRuntimeState.js index cf2ee0c389..c197be4b83 100644 --- a/ZelBack/src/services/appManagement/appsRuntimeState.js +++ b/ZelBack/src/services/appManagement/appsRuntimeState.js @@ -50,20 +50,17 @@ function canonical(identifier) { /** * Returns the persisted runtime-state document for a component identifier - * (e.g. `component_app`, or the app name for v1-3 apps), or null. + * (e.g. `component_app`, or the app name for v1-3 apps), or null when it has + * none. Rejects when the read fails: an unread record is not an absent one, and + * every lock and ladder this document holds would read as clear. * * @param {string} identifier * @returns {Promise} */ async function getState(rawIdentifier) { const identifier = canonical(rawIdentifier); - try { - const database = collection(); - return await dbHelper.findOneInDatabase(database, appsRuntimeState, { identifier }, { projection: { _id: 0 } }); - } catch (err) { - log.error(`appsRuntimeState - failed to read state for ${identifier}: ${err.message}`); - return null; - } + const database = collection(); + return dbHelper.findOneInDatabase(database, appsRuntimeState, { identifier }, { projection: { _id: 0 } }); } function isDuplicateKeyError(err) { @@ -149,9 +146,9 @@ async function setOperatorStopped(identifier, stopped, opts = {}) { await setFields(identifier, fields); } -// The stop lock and its start request, read without getState's swallow: these -// readers decide whether to lift the lock, and a read failure answered as "no -// record" is a lock lifted on no evidence. +// The stop lock and its start request, the only fields these readers need. +// Rejects when the read fails: a lock lifted on an unread record is lifted on no +// evidence. function readLock(identifier) { return dbHelper.findOneInDatabase( collection(), @@ -162,9 +159,8 @@ function readLock(identifier) { } /** - * Whether the operator stop lock is on, read without getState's swallow: for a - * caller about to commit a component to running, an unread lock is not an - * absent one. + * Whether the operator stop lock is on: for a caller about to commit a component + * to running, an unread lock is not an absent one. * * @param {string} rawIdentifier * @returns {Promise} @@ -298,26 +294,9 @@ async function recordRestartGeneration(identifier, generation) { } /** - * Whether the operator has deliberately stopped this component, so the - * reconciler (and the masterSlave/syncthing deciders) must leave it stopped. - * - * @param {string} identifier - * @returns {Promise} - */ -async function isOperatorStopped(identifier) { - const state = await getState(identifier); - return state?.operatorStopped === true; -} - -/** - * The operator's stop lock and the mode they asked for, from ONE read. - * - * They are two fields of one document, and a caller needing both must not ask - * twice: getState returns null for a read failure exactly as it does for "no - * record", so a second read that failed reported no force flag and turned the - * operator's "kill now" into a drain they did not ask for. Answering both from a - * single read is a window that cannot open - and one round-trip fewer on every - * stop the reconciler performs. + * The operator's stop lock and the mode they asked for, from ONE read: two + * fields of one document, answered together so they always describe the same + * moment. Rejects when the read fails. * * `startRequested` is a start the operator has asked for and the election has * not yet decided; the lock stands until it has. `asPrimary` is whether this @@ -638,18 +617,15 @@ async function removeControllerState(rawIdentifier) { } /** - * Drops all runtime state for a component (on uninstall). + * Drops all runtime state for a component (on uninstall). Rejects when the + * delete fails: a stop lock left behind holds a later reinstall stopped. * * @param {string} identifier */ async function remove(rawIdentifier) { const identifier = canonical(rawIdentifier); - try { - const database = collection(); - await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier }); - } catch (err) { - log.error(`appsRuntimeState - failed to remove state for ${identifier}: ${err.message}`); - } + const database = collection(); + await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier }); } /** @@ -710,7 +686,7 @@ async function getVolumeImage(rawIdentifier) { async function prepareCollection() { try { const database = collection(); - const all = await dbHelper.findInDatabase(database, appsRuntimeState, {}, { projection: { _id: 0 } }); + const all = await dbHelper.findInDatabase(database, appsRuntimeState, {}, {}); const byIdentifier = new Map(); // eslint-disable-next-line no-restricted-syntax for (const doc of all) { @@ -770,10 +746,13 @@ async function prepareCollection() { merged.lastDiedAt = newestExit.lastDiedAt; } log.warn(`appsRuntimeState - merged ${twins.length} duplicate docs for ${identifier} (operatorStopped=${merged.operatorStopped}, ${merged.restartHistory.length} restart entries)`); + // The merge lands on one twin before the others go, so a failure part way + // leaves duplicates for the next start to merge, never no record at all. + const kept = oldestFirst[oldestFirst.length - 1]._id; // eslint-disable-next-line no-await-in-loop - await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier }); + await dbHelper.updateOneInDatabase(database, appsRuntimeState, { _id: kept }, { $set: merged }); // eslint-disable-next-line no-await-in-loop - await dbHelper.updateOneInDatabase(database, appsRuntimeState, { identifier }, { $set: merged }, { upsert: true }); + await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier, _id: { $ne: kept } }); } } await database.collection(appsRuntimeState).createIndex({ identifier: 1 }, { unique: true, name: 'identifier_unique' }); @@ -789,7 +768,6 @@ module.exports = { requestOperatorStart, releaseOperatorStart, operatorStoppedOrThrow, - isOperatorStopped, operatorStopState, operatorHeldIdentifiers, recordRestart, diff --git a/ZelBack/src/services/appMonitoring/appReconciler.js b/ZelBack/src/services/appMonitoring/appReconciler.js index b97f2249b8..09f9ede44d 100644 --- a/ZelBack/src/services/appMonitoring/appReconciler.js +++ b/ZelBack/src/services/appMonitoring/appReconciler.js @@ -375,6 +375,8 @@ function isManagedElsewhere(identifier) { const restore = globalState.restoreInProgress || []; if (backup.includes(mainAppName) || restore.includes(mainAppName)) return true; if (globalState.stoppingContainers.has(dockerService.getAppIdentifier(identifier))) return true; + // an app being removed, including one whose removal is still cleaning up + if (globalState.departingApps.has(mainAppName)) return true; return false; } @@ -382,11 +384,8 @@ function isManagedElsewhere(identifier) { * What this component should be doing, why, and - when the answer is an operator * stop - whether that operator asked for a hard kill. * - * The lock and the force flag are fields of one document, so they come from one - * read. The stop branch used to re-read the same document for the flag alone, - * and getState returns null for a read failure exactly as it does for "no - * record" - so a second read that failed turned "kill now" into a drain. One - * read is a window that cannot open, and one round-trip fewer on every stop. + * The lock and the force flag come from one read. A read that fails rejects, + * and the pass is retried without acting: an unread lock is not a lifted one. * * @returns {Promise<{desired: boolean|null, reason: string, force: boolean}>} */ @@ -1106,8 +1105,9 @@ async function reconcile(rawIdentifier) { // without it - which is how far a component has escalated, and whether it // ever went backwards. Read on the backoff path only, which is a paced // restart and therefore cold. - const backoffState = await appsRuntimeState.getState(identifier); - const rung = ((backoffState && backoffState.restartHistory) || []).length; + // For the report only: a failed read leaves the rung unknown, never the wait. + const backoffState = await appsRuntimeState.getState(identifier).catch(() => undefined); + const rung = backoffState === undefined ? null : ((backoffState && backoffState.restartHistory) || []).length; log.warn(`appReconciler - ${identifier} stopped, ${cause}; backing off ${Math.round(wait / 1000)}s before restart (rung ${rung})`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'backoff', waitMs: wait, rung, crashed, @@ -1122,6 +1122,13 @@ async function reconcile(rawIdentifier) { const isComponent = spec.appSpec.version >= 4 && Array.isArray(spec.appSpec.compose); await volumeService.ensureMountPathsExist(spec.comp, mainAppName, isComponent, isComponent ? spec.appSpec : null); + // The restart this start satisfies is the one requested before it: read now, so + // a failed read defers before anything is started, and a restart requested + // while the start runs is still owed afterwards. + const startingState = await appsRuntimeState.getState(identifier); + const pendingGeneration = (startingState && startingState.restartGeneration) || 0; + const satisfiesRestart = pendingGeneration > ((startingState && startingState.actuatedRestartGeneration) || 0); + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.RECONCILER_BEFORE_START, identifier); // A decider writes its opinion outside this single-flight, so one may have been // written during the awaits above; starting on what this pass read would undo it @@ -1159,9 +1166,6 @@ async function reconcile(rawIdentifier) { // A restart of a container that was already stopped IS this start, so the // request is satisfied here. Left pending, the pass that next finds it running // would bounce a container the operator has just watched come up. - const startedState = await appsRuntimeState.getState(identifier); - const pendingGeneration = (startedState && startedState.restartGeneration) || 0; - const satisfiesRestart = pendingGeneration > ((startedState && startedState.actuatedRestartGeneration) || 0); log.info(`appReconciler - ${identifier} restarted`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'started', exitCode: actual.exitCode }); // A start is exactly when a container can come up attached to no network (a stale diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 9d83e8faa1..c8b4ced5c7 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -1063,6 +1063,18 @@ describe('advancedWorkflows tests', () => { expect(linesMatching(logInfo, 'operator-stopped')).to.have.lengthOf(1); }); + it('excludes a component whose stop lock cannot be read, and elects nothing for it', async () => { + const appName = 'unreadlockapp'; + sinon.stub(appsRuntimeState, 'operatorStopState').rejects(new Error('connection timed out')); + const logWarn = sinon.stub(log, 'warn'); + const runPass = electionFixture(appName); + + await runPass(); + + expect(serviceHelperStub.called, 'the election went on to FDM with an unread lock').to.be.false; + expect(linesMatching(logWarn, 'stop lock could not be read')).to.have.lengthOf(1); + }); + it('announces again after the operator lock is lifted and re-applied', async () => { const appName = 'relockapp'; const operatorStopped = sinon.stub(appsRuntimeState, 'operatorStopState'); @@ -2783,7 +2795,6 @@ describe('advancedWorkflows tests', () => { // back within it keeps what it holds. it('will not start beside a silent peer that announced a shutdown and has not passed its expiry', async () => { const appName = 'rebootingpeerapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2813,7 +2824,6 @@ describe('advancedWorkflows tests', () => { it('will not start beside a silent peer when this node cannot read which nodes announced a shutdown', async () => { const appName = 'shutdownsunreadableapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2871,7 +2881,6 @@ describe('advancedWorkflows tests', () => { // resumes them. The closed connection is this node's doing, not the peer's. it('will not start beside a silent peer whose device this node\'s syncthing has paused', async () => { const appName = 'pausedpeerapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2888,7 +2897,6 @@ describe('advancedWorkflows tests', () => { // that closed before it started says nothing about the peer now. it('will not start beside a silent peer whose connection closed before this node\'s syncthing started', async () => { const appName = 'restartedsyncthingapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2905,7 +2913,6 @@ describe('advancedWorkflows tests', () => { // as the previous syncthing stopped reads the same second as this one's start. it('will not start beside a silent peer whose connection closed in the second this node\'s syncthing started', async () => { const appName = 'samesecondapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -3016,7 +3023,6 @@ describe('advancedWorkflows tests', () => { it('will not start beside a silent peer when this node cannot read when its syncthing started', async () => { const appName = 'statusunreadableapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -3031,7 +3037,6 @@ describe('advancedWorkflows tests', () => { it('will not start beside a silent peer when this node cannot read whether its syncthing paused it', async () => { const appName = 'devicesunreadableapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -3367,7 +3372,6 @@ describe('advancedWorkflows tests', () => { // the node it names may not be the one running it. it('lets a standby\'s paused folder ask every other holder, not only the node FDM names', async () => { const appName = 'fdmholderapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); sinon.stub(log, 'info'); const hold = sinon.stub(primaryRole, 'holdAsStandby').resolves(); const runPass = electionFixture(appName, ['192.168.1.90:16137', '192.168.1.91:16127']); diff --git a/tests/unit/appReconciler.test.js b/tests/unit/appReconciler.test.js index 7927fb1cdf..f22d7ee545 100644 --- a/tests/unit/appReconciler.test.js +++ b/tests/unit/appReconciler.test.js @@ -62,7 +62,6 @@ describe('appReconciler tests', () => { clearAppVolumeData: sinon.stub().resolves(), }, appsRuntimeState: { - isOperatorStopped: sinon.stub().resolves(false), operatorStopState: sinon.stub().resolves({ stopped: false, force: false }), operatorStoppedOrThrow: sinon.stub().resolves(false), restartWaitMs: sinon.stub().resolves(0), @@ -243,12 +242,8 @@ describe('appReconciler tests', () => { expect(stubs.dockerService.appDockerKill.called).to.be.false; }); - // The defect this closes: the lock and the mode were two reads of one - // document, and getState returns null for a read failure exactly as it does - // for "no record" - so a second read that failed reported no force flag and - // downgraded the operator's kill into a drain they never asked for. The mode - // now arrives with the decision that read it, so nothing later can contradict - // it. An empty getState here IS what that failed read looked like. + // The lock and the mode arrive together from the read that decided the stop, + // so no later read of the same document can contradict them. it('kills even when a later state read comes back empty', async () => { stubs.appsRuntimeState.operatorStopState.resolves({ stopped: true, force: true }); stubs.appsRuntimeState.getState.resolves(null); @@ -449,6 +444,35 @@ describe('appReconciler tests', () => { ).to.equal(false); }); + it('leaves a component alone while its app is departing, removal cleanup included', async () => { + stubs.globalState.departingApps.enter('App'); + + await appReconciler.reconcile('www_App'); + + expect(stubs.dockerService.dockerContainerInspect.called, 'a departing app was reconciled').to.be.false; + expect(stubs.dockerService.appDockerStart.called).to.be.false; + stubs.globalState.departingApps.leave('App'); + }); + + it('starts nothing when the stop lock cannot be read, and fails the pass for a retry', async () => { + stubs.appsRuntimeState.operatorStopState.rejects(new Error('connection timed out')); + + const failure = await appReconciler.reconcile('www_App').catch((err) => err); + + expect(failure, 'the pass reported success on an unread lock').to.be.an('error'); + expect(stubs.dockerService.appDockerStart.called).to.be.false; + }); + + it('starts nothing when the restart generation cannot be read, and fails the pass for a retry', async () => { + stubs.appsRuntimeState.getState.rejects(new Error('connection timed out')); + + const failure = await appReconciler.reconcile('www_App').catch((err) => err); // inspect default: stopped + + expect(failure, 'the pass reported success on an unread generation').to.be.an('error'); + expect(stubs.dockerService.appDockerStart.called).to.be.false; + expect(stubs.appsRuntimeState.recordRestart.called, 'no attempt was made, so none is recorded').to.be.false; + }); + it('starts a stopped plain component that should run (default always policy)', async () => { await appReconciler.reconcile('www_App'); expect(stubs.appsRuntimeState.recordRestart.calledOnceWith('www_App')).to.be.true; @@ -2213,7 +2237,7 @@ describe('appReconciler tests', () => { }); describe('applyIntent serialises an intent write against a reconcile pass', () => { - // The defect this closes: a pass reads isOperatorStopped, then acts on that + // The defect this closes: a pass reads the stop lock, then acts on that // answer once docker has replied. An operator stop landing in that gap is // not seen, so the pass starts a container the operator has just stopped and // the next pass stops it again - the flap suite 45 catches. diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index ed5bfc9deb..88156509c3 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -595,7 +595,7 @@ describe('appUninstaller tests', () => { // and unforced paths alike. let runtimeStateStub; - function buildUninstaller(spec, constantOverrides = {}) { + function buildUninstaller(spec, constantOverrides = {}, dbOverrides = {}) { runtimeStateStub = { remove: sinon.stub().resolves(), removeControllerState: sinon.stub().resolves() }; // The REAL departing tracker, taken fresh per test rather than // reimplemented here: a fake that counts differently from the module @@ -635,6 +635,7 @@ describe('appUninstaller tests', () => { findOneInDatabase: sinon.stub().resolves(spec), findInDatabase: sinon.stub().resolves([]), findOneAndDeleteInDatabase: sinon.stub().resolves(), + ...dbOverrides, }, '../dockerService': { appDockerKill: sinon.stub().resolves(), @@ -721,6 +722,43 @@ describe('appUninstaller tests', () => { const res = null; // exercised without a response stream + describe('a cleanup step that fails after the containers are gone', () => { + let clock; + + afterEach(() => { + clock?.restore(); + clock = undefined; + }); + + it('holds the app as departing until its row is deleted, retrying the delete', async () => { + const deleteRow = sinon.stub(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + deleteRow.resolves(); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + + await uninstaller.removeAppLocally('testapp', res, true); + + expect(globalStateStub.departingApps.has('testapp'), 'a row that outlived its containers is left for the reconciler').to.be.true; + await clock.tickAsync(30 * 1000); + sinon.assert.calledTwice(deleteRow); + expect(globalStateStub.departingApps.has('testapp')).to.be.false; + }); + + it('holds the app as departing until its runtime state is removed, retrying the removal', async () => { + const uninstaller = buildUninstaller(v2Spec); + runtimeStateStub.remove.onFirstCall().rejects(new Error('connection timed out')); + clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + + await uninstaller.removeAppLocally('testapp', res, true); + + expect(globalStateStub.departingApps.has('testapp')).to.be.true; + await clock.tickAsync(30 * 1000); + sinon.assert.calledTwice(runtimeStateStub.remove); + expect(globalStateStub.departingApps.has('testapp')).to.be.false; + }); + }); + it('notifies the seam with the bare name for a v1-3 app', async () => { const uninstaller = buildUninstaller(v2Spec); const onRemoved = sinon.stub(); diff --git a/tests/unit/appsRuntimeState.test.js b/tests/unit/appsRuntimeState.test.js index 1b8aa737db..29eeefa3d3 100644 --- a/tests/unit/appsRuntimeState.test.js +++ b/tests/unit/appsRuntimeState.test.js @@ -7,10 +7,12 @@ describe('appsRuntimeState tests', () => { let store; // fake collection: identifier -> doc let logStub; let readFails; + let removeFails; beforeEach(() => { store = new Map(); readFails = false; + removeFails = false; logStub = { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }; const dbHelperStub = { @@ -63,7 +65,10 @@ describe('appsRuntimeState tests', () => { }); store.set(query.identifier, next); }, - removeDocumentsFromCollection: async (_db, _coll, query) => { store.delete(query.identifier); }, + removeDocumentsFromCollection: async (_db, _coll, query) => { + if (removeFails) throw new Error('connection timed out'); + store.delete(query.identifier); + }, }; appsRuntimeState = proxyquire('../../ZelBack/src/services/appManagement/appsRuntimeState', { @@ -112,11 +117,11 @@ describe('appsRuntimeState tests', () => { describe('operatorStopped', () => { it('persists the stop lock and reads it back', async () => { await appsRuntimeState.setOperatorStopped('www_App', true); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; }); it('defaults to not-stopped for an unknown component', async () => { - expect(await appsRuntimeState.isOperatorStopped('nope_App')).to.be.false; + expect(await appsRuntimeState.operatorStoppedOrThrow('nope_App')).to.be.false; }); it('clearing the lock (start) also clears the restart backoff history', async () => { @@ -125,7 +130,7 @@ describe('appsRuntimeState tests', () => { expect(store.get('www_App').restartHistory).to.have.lengthOf(2); await appsRuntimeState.setOperatorStopped('www_App', false); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.false; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.false; expect(store.get('www_App').restartHistory).to.deep.equal([]); }); @@ -237,7 +242,7 @@ describe('appsRuntimeState tests', () => { expect(await appsRuntimeState.requestOperatorStart('www_App')).to.be.true; - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; expect(await appsRuntimeState.operatorStopState('www_App')).to.deep.equal({ stopped: true, startRequested: true, force: false, asPrimary: true, }); @@ -272,7 +277,7 @@ describe('appsRuntimeState tests', () => { expect(await appsRuntimeState.releaseOperatorStart('www_App')).to.be.false; - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; }); it('leaves a lock the operator set again after asking for the start', async () => { @@ -282,7 +287,7 @@ describe('appsRuntimeState tests', () => { expect((await appsRuntimeState.operatorStopState('www_App')).startRequested).to.be.false; expect(await appsRuntimeState.releaseOperatorStart('www_App')).to.be.false; - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; }); it('does not report a start request once the lock is gone', async () => { @@ -328,7 +333,32 @@ describe('appsRuntimeState tests', () => { expect(requested, 'an unread lock was answered').to.be.an('error'); expect(released, 'an unread lock was answered').to.be.an('error'); readFails = false; - expect(await appsRuntimeState.isOperatorStopped('www_App'), 'the lock was lifted').to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App'), 'the lock was lifted').to.be.true; + }); + }); + + describe('a record that cannot be read', () => { + it('rejects rather than answering no record', async () => { + await appsRuntimeState.setOperatorStopped('www_App', true); + readFails = true; + + const state = await appsRuntimeState.getState('www_App').catch((err) => err); + const stop = await appsRuntimeState.operatorStopState('www_App').catch((err) => err); + + expect(state, 'an unread record was answered').to.be.an('error'); + expect(stop, 'an unread lock was answered').to.be.an('error'); + }); + + it('records no restart over a ladder it could not read', async () => { + await appsRuntimeState.recordRestart('www_App'); + await appsRuntimeState.recordRestart('www_App'); + const ladder = store.get('www_App').restartHistory; + readFails = true; + + await appsRuntimeState.recordRestart('www_App'); + + readFails = false; + expect(store.get('www_App').restartHistory).to.deep.equal(ladder); }); }); @@ -388,6 +418,16 @@ describe('appsRuntimeState tests', () => { await appsRuntimeState.remove('www_App'); expect(store.has('www_App')).to.be.false; }); + + it('remove rejects when the delete fails, rather than leaving the stop lock in silence', async () => { + await appsRuntimeState.setOperatorStopped('www_App', true); + removeFails = true; + + const failure = await appsRuntimeState.remove('www_App').catch((err) => err); + + expect(failure).to.be.an('error'); + expect(store.get('www_App').operatorStopped).to.be.true; + }); }); describe('backoff ladder reset requires a stable RUN (not time since the attempt)', () => { @@ -663,14 +703,14 @@ describe('appsRuntimeState tests', () => { // The module therefore normalizes at its own boundary. it('keys a docker-prefixed identifier and its bare form to the same document', async () => { await appsRuntimeState.setOperatorStopped('fluxwww_App', true); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; expect(store.has('fluxwww_App')).to.be.false; }); it('removes under either form', async () => { await appsRuntimeState.setOperatorStopped('www_App', true); await appsRuntimeState.remove('fluxwww_App'); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.false; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.false; }); }); @@ -961,6 +1001,39 @@ describe('appsRuntimeState tests', () => { // here survives only by being carried, from whichever twin holds it. The // field names below are deliberately not real ones - a real one would be // testing its own named rule instead. + it('writes the merge onto the newest twin before removing the others, and removes nothing when that write fails', async () => { + const ops = []; + let writeFails = false; + const orderState = proxyquire('../../ZelBack/src/services/appManagement/appsRuntimeState', { + '../../lib/log': logStub, + '../dbHelper': { + databaseConnection: () => ({ db: () => ({ collection: () => ({ createIndex: sinon.stub().resolves() }) }) }), + findInDatabase: async () => [ + { _id: 'older', identifier: 'www_App', operatorStopped: true, updatedAt: 1000 }, + { _id: 'newer', identifier: 'www_App', restartHistory: [5], updatedAt: 9000 }, + ], + removeDocumentsFromCollection: async (_db, _coll, query) => { ops.push(['remove', query]); }, + updateOneInDatabase: async (_db, _coll, query, update) => { + if (writeFails) throw new Error('connection timed out'); + ops.push(['write', query, update.$set]); + }, + }, + '../dockerService': { getBaseAppName: (id) => id }, + }); + + await orderState.prepareCollection(); + + expect(ops.map(([op]) => op)).to.deep.equal(['write', 'remove']); + expect(ops[0][1]).to.deep.equal({ _id: 'newer' }); + expect(ops[0][2].operatorStopped, 'the lock on the older twin').to.be.true; + expect(ops[1][1]).to.deep.equal({ identifier: 'www_App', _id: { $ne: 'newer' } }); + + ops.length = 0; + writeFails = true; + await orderState.prepareCollection(); + expect(ops, 'a twin was removed although the merge was never written').to.deep.equal([]); + }); + it('carries fields the merge does not name, from whichever twin has them', async () => { docs = [ { diff --git a/tests/unit/fixtures/globalState.js b/tests/unit/fixtures/globalState.js index e73d4143a4..d8fab64de5 100644 --- a/tests/unit/fixtures/globalState.js +++ b/tests/unit/fixtures/globalState.js @@ -44,7 +44,8 @@ function resetGlobalState() { // so they are emptied that way below rather than written to here. else if (Array.isArray(value)) { if (!Object.isFrozen(value)) value.length = 0; } else if (value && typeof value === 'object' && value.constructor === Object) { - for (const own of Object.keys(value)) delete value[own]; + // An object's functions are its interface, not its state. + for (const own of Object.keys(value)) if (typeof value[own] !== 'function') delete value[own]; } } for (const appname of globalState.backupInProgress) globalState.finishBackup(appname); From aeecfdf6653d97515f7d4169695571771182f4dc Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 10:52:56 +0100 Subject: [PATCH 048/369] fix(db): a database that does not answer never uninstalls an app A database that did not answer says nothing about the app, so it is never the reason an app is uninstalled. dbHelper.isDatabaseError tells such a failure from the work around it. - a missing container whose recreate fails on a database read is retried, not removed and recorded as tampering - a soft redeploy whose next step cannot read the database leaves the app down with its volume, as a declined reinstall does - a soft or hard component redeploy does not uninstall the whole app, and every other component's data, over a database read - the reinstall pass leaves an app whose redeploy stopped on the database for a later pass - a soft registration stops where it is and reports the new install outcome DEFERRED: nothing torn down, no claim about the app - a hard redeploy that fails before its teardown leaves the running app alone appsResources rejects with the read's own error instead of answering an error envelope, so the requirements checks fail with it and the redeploy paths can tell what failed; /apps/appsresources answers the error as before. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 57 +++++++- .../services/appMonitoring/appReconciler.js | 7 + .../services/appQuery/resourceQueryService.js | 127 +++++++++--------- ZelBack/src/services/dbHelper.js | 13 ++ ZelBack/src/services/fluxService.js | 3 - ZelBack/src/services/utils/installOutcome.js | 7 +- tests/unit/advancedWorkflows.test.js | 81 +++++++++++ tests/unit/appReconciler.test.js | 15 +++ tests/unit/fluxService.test.js | 6 +- tests/unit/resourceQueryService.test.js | 10 ++ 10 files changed, 251 insertions(+), 75 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 81b9f307cb..0b655db5e1 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -1224,6 +1224,12 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { res.write(serviceHelper.ensureString(errorResponse)); if (res.flush) res.flush(); } + if (dbHelper.isDatabaseError(error)) { + // A soft registration keeps the app's data, and the teardown below would not: + // a database that did not answer is no reason to destroy it. + log.warn(`Soft registration of ${appSpecs.name} stopped: the database did not answer (${error.message}). Nothing is torn down.`); + return InstallOutcome.DEFERRED; + } const removeStatus = messageHelper.createErrorMessage(`Error occured. Initiating Flux App ${appSpecs.name} removal`); log.info(removeStatus); log.warn(`REMOVAL REASON: Soft registration failure - ${appSpecs.name} failed during soft registration: ${error.message} (softRegisterAppLocally)`); @@ -1634,6 +1640,20 @@ async function softRedeploy(appSpecs, res) { } return; } + if (dbHelper.isDatabaseError(error)) { + // The app is down and its volume is intact. A database that did not answer + // says nothing about the app, so it is left as a declined reinstall leaves + // it, never uninstalled at the cost of its data. + const notReinstalled = messageHelper.createErrorMessage( + `Application ${appSpecs.name} was not reinstalled: the database did not answer (${error.message})`, + ); + log.warn(notReinstalled); + if (res) { + res.write(serviceHelper.ensureString(notReinstalled)); + if (res.flush) res.flush(); + } + return; + } log.warn(`REMOVAL REASON: Soft redeploy failure - ${appSpecs.name} failed during soft redeploy: ${error.message} (softRedeploy)`); // eslint-disable-next-line global-require const appUninstaller = require('./appUninstaller'); @@ -1651,6 +1671,9 @@ async function softRedeploy(appSpecs, res) { async function hardRedeploy(appSpecs, res) { // eslint-disable-next-line global-require const appUninstaller = require('./appUninstaller'); + // Whether the teardown was reached. A failure before it leaves a running app + // running: the forced uninstall below answers only a failed rebuild. + let teardownStarted = false; try { // Every operation, including the periodic reinstall pass - which sets its // flag before it tears anything down and holds it across the wait, so the @@ -1670,6 +1693,7 @@ async function hardRedeploy(appSpecs, res) { globalState.hardRedeployInProgress = true; log.warn(`REMOVAL REASON: Hard redeploy initiated - ${appSpecs.name} being removed as part of hard redeploy process (hardRedeploy)`); + teardownStarted = true; await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); const appRedeployResponse = messageHelper.createSuccessMessage('Application removed. Awaiting installation...'); log.info(appRedeployResponse); @@ -1699,8 +1723,19 @@ async function hardRedeploy(appSpecs, res) { globalState.hardRedeployInProgress = false; } catch (error) { log.error(error); - log.warn(`REMOVAL REASON: Hard redeploy failure - ${appSpecs.name} failed during hard redeploy: ${error.message} (hardRedeploy)`); globalState.hardRedeployInProgress = false; + if (!teardownStarted) { + const notStarted = messageHelper.createErrorMessage( + `Hard redeploy of ${appSpecs.name} failed before the app was taken down: ${error.message}`, + ); + log.warn(notStarted); + if (res) { + res.write(serviceHelper.ensureString(notStarted)); + if (res.flush) res.flush(); + } + return; + } + log.warn(`REMOVAL REASON: Hard redeploy failure - ${appSpecs.name} failed during hard redeploy: ${error.message} (hardRedeploy)`); // endResponse false: redeployAPI opened this response and closes it. await appUninstaller.removeAppLocally(appSpecs.name, res, true, false, true); log.info(`Cleanup completed for ${appSpecs.name} after hard redeploy failure`); @@ -1862,6 +1897,12 @@ async function softRedeployComponent(appName, componentName, res) { + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.'); throw error; } + if (dbHelper.isDatabaseError(error)) { + // Uninstalling the whole app takes every other component's data with it, + // and a database that did not answer says nothing about any of them. + log.warn(`Component ${fullComponentName} was not reinstalled: the database did not answer (${error.message}). The app is not uninstalled.`); + throw error; + } log.warn(`REMOVAL REASON: Soft redeploy failure - ${appName} being removed after component ${fullComponentName} failed during soft redeploy: ${error.message} (softRedeployComponent)`); // endResponse false: the endpoint opened this response and closes it. await appUninstaller.removeAppLocally(appName, res, true, false, true); @@ -2006,6 +2047,12 @@ async function hardRedeployComponent(appName, componentName, res) { + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.'); throw error; } + if (dbHelper.isDatabaseError(error)) { + // Uninstalling the whole app takes every other component's data with it, + // and a database that did not answer says nothing about any of them. + log.warn(`Component ${fullComponentName} was not reinstalled: the database did not answer (${error.message}). The app is not uninstalled.`); + throw error; + } log.warn(`REMOVAL REASON: Hard redeploy failure - ${appName} being removed after component ${fullComponentName} failed during hard redeploy: ${error.message} (hardRedeployComponent)`); // endResponse false: the endpoint opened this response and closes it. await appUninstaller.removeAppLocally(appName, res, true, false, true); @@ -4646,6 +4693,14 @@ async function reinstallOldApplications() { } } catch (error) { log.error(error); + if (dbHelper.isDatabaseError(error)) { + // The components' data is still on disk, and a database that did not + // answer says nothing about the app: it is left for a later pass, never + // uninstalled with its data. + log.warn(`Redeployment of ${appSpecifications.name} stopped: the database did not answer (${error.message}). The app is not uninstalled.`); + // eslint-disable-next-line no-continue + continue; + } log.warn(`REMOVAL REASON: Redeployment error - ${appSpecifications.name} failed during redeployment: ${error.message}`); // eslint-disable-next-line no-await-in-loop await appUninstaller.removeAppLocally(appSpecifications.name, null, true, true, true); // remove entire app diff --git a/ZelBack/src/services/appMonitoring/appReconciler.js b/ZelBack/src/services/appMonitoring/appReconciler.js index 09f9ede44d..ac409bed64 100644 --- a/ZelBack/src/services/appMonitoring/appReconciler.js +++ b/ZelBack/src/services/appMonitoring/appReconciler.js @@ -484,6 +484,13 @@ async function recreateMissing(identifier) { scheduleRetry(identifier, MANAGED_RETRY_MS); return; } + // A database that did not answer says nothing about the container: the + // recreate never got as far as building it. + if (dbHelper.isDatabaseError(err)) { + log.warn(`appReconciler - recreate of ${identifier} could not read what it needs (${err.message}); retrying, not removing`); + scheduleRetry(identifier, MANAGED_RETRY_MS); + return; + } log.error(`appReconciler - failed to recreate ${identifier}: ${err.message}`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'recreateFailed', reason: err.message }); await appTamperingDetectionService.recordEvent(mainAppName, 'recreation_failed', `Container recreation failure: ${err.message}`); diff --git a/ZelBack/src/services/appQuery/resourceQueryService.js b/ZelBack/src/services/appQuery/resourceQueryService.js index 3e797fd08c..2ad6938b2b 100644 --- a/ZelBack/src/services/appQuery/resourceQueryService.js +++ b/ZelBack/src/services/appQuery/resourceQueryService.js @@ -75,72 +75,65 @@ async function fluxUsage(req, res) { * of them means this node cannot account for itself. It does not leave the * process: the API halves publish the totals alone. * - * @returns {object} Message carrying {appsCpusLocked, appsRamLocked, appsHddLocked, unreadable} + * @returns {object} Message carrying {appsCpusLocked, appsRamLocked, appsHddLocked, unreadable}. + * Rejects with the original error when the installed apps cannot be read: a + * node that cannot count what it has committed has no totals to give. */ async function appsResources() { log.info('Checking appsResources'); - try { - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const appsQuery = {}; - const appsProjection = { projection: { _id: 0 } }; - const stored = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); - - // eslint-disable-next-line global-require - const { decryptEnterpriseApps } = require('./appQueryService'); - const { inPlace: appsResult, unreadable } = await decryptEnterpriseApps(stored, { formatSpecs: false }); - - let appsCpusLocked = 0; - let appsRamLocked = 0; - let appsHddLocked = 0; - const tier = await generalService.nodeTier().catch((error) => log.error(error)); - const hddTier = `hdd${tier}`; - const ramTier = `ram${tier}`; - const cpuTier = `cpu${tier}`; - - // Ensure appsResult is an array - const apps = Array.isArray(appsResult) ? appsResult : []; - apps.forEach((app) => { - if (app.version >= 4) { - app.compose.forEach((component) => { - if (component.tiered && tier) { - appsCpusLocked += serviceHelper.ensureNumber(component[cpuTier] || component.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(component[ramTier] || component.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(component[hddTier] || component.hdd) || 0; - } else { - appsCpusLocked += serviceHelper.ensureNumber(component.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(component.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(component.hdd) || 0; - } - appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap - }); - } else if (app.tiered && tier) { - appsCpusLocked += serviceHelper.ensureNumber(app[cpuTier] || app.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(app[ramTier] || app.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(app[hddTier] || app.hdd) || 0; - appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap - } else { - appsCpusLocked += serviceHelper.ensureNumber(app.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(app.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(app.hdd) || 0; + const dbopen = dbHelper.databaseConnection(); + const appsDatabase = dbopen.db(config.database.appslocal.database); + const appsQuery = {}; + const appsProjection = { projection: { _id: 0 } }; + const stored = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); + + // eslint-disable-next-line global-require + const { decryptEnterpriseApps } = require('./appQueryService'); + const { inPlace: appsResult, unreadable } = await decryptEnterpriseApps(stored, { formatSpecs: false }); + + let appsCpusLocked = 0; + let appsRamLocked = 0; + let appsHddLocked = 0; + const tier = await generalService.nodeTier().catch((error) => log.error(error)); + const hddTier = `hdd${tier}`; + const ramTier = `ram${tier}`; + const cpuTier = `cpu${tier}`; + + // Ensure appsResult is an array + const apps = Array.isArray(appsResult) ? appsResult : []; + apps.forEach((app) => { + if (app.version >= 4) { + app.compose.forEach((component) => { + if (component.tiered && tier) { + appsCpusLocked += serviceHelper.ensureNumber(component[cpuTier] || component.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(component[ramTier] || component.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(component[hddTier] || component.hdd) || 0; + } else { + appsCpusLocked += serviceHelper.ensureNumber(component.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(component.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(component.hdd) || 0; + } appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap - } - }); - const appsUsage = { - appsCpusLocked, - appsRamLocked, - appsHddLocked, - unreadable: unreadable.map((app) => app.name), - }; - return messageHelper.createDataMessage(appsUsage); - } catch (error) { - log.error(error); - return messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - } + }); + } else if (app.tiered && tier) { + appsCpusLocked += serviceHelper.ensureNumber(app[cpuTier] || app.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(app[ramTier] || app.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(app[hddTier] || app.hdd) || 0; + appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap + } else { + appsCpusLocked += serviceHelper.ensureNumber(app.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(app.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(app.hdd) || 0; + appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap + } + }); + const appsUsage = { + appsCpusLocked, + appsRamLocked, + appsHddLocked, + unreadable: unreadable.map((app) => app.name), + }; + return messageHelper.createDataMessage(appsUsage); } /** @@ -179,10 +172,12 @@ function unaccountedApps(response) { * @returns {Promise} */ async function appsResourcesApi(req, res) { - const response = await appsResources(); - - if (response.status === 'error') { - res.json(response); + let response; + try { + response = await appsResources(); + } catch (error) { + log.error(error); + res.json(messageHelper.createErrorMessage(error.message || error, error.name, error.code)); return; } diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index ff8be7fc53..d9c346649b 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -34,6 +34,18 @@ function databaseConnection() { return openDBConnection; } +/** + * Whether an error came from the database rather than from the work around it: + * a timeout, a pool wait, an unreachable server, or a refused command. A caller + * deciding whether a failure says anything about an app reads this - a database + * that did not answer says nothing about the app. + * @param {*} error + * @returns {boolean} + */ +function isDatabaseError(error) { + return error instanceof mongodb.MongoError; +} + /** * Initiates connection with the database. * @@ -1127,6 +1139,7 @@ if (require.main === module) { module.exports = { aggregateInDatabase, + isDatabaseError, appMessagesThatCount, appPaymentFacts, bulkWriteInDatabase, diff --git a/ZelBack/src/services/fluxService.js b/ZelBack/src/services/fluxService.js index 573ffb2297..390bb0cdaa 100644 --- a/ZelBack/src/services/fluxService.js +++ b/ZelBack/src/services/fluxService.js @@ -1658,9 +1658,6 @@ async function getFluxInfo(req, res) { // at one exit and not the other is the shape it exists to avoid. info.apps.runningapps = appQueryService.publicContainerView(appsRunning.data); const appsResources = await resourceQueryService.appsResources(); - if (appsResources.status === 'error') { - throw appsResources.data; - } // The same three numbers /apps/appsresources publishes. This endpoint embeds // them, and a field kept from one exit and not the other is the shape that // catches this file out. diff --git a/ZelBack/src/services/utils/installOutcome.js b/ZelBack/src/services/utils/installOutcome.js index 7248edcec1..d638c3b5ed 100644 --- a/ZelBack/src/services/utils/installOutcome.js +++ b/ZelBack/src/services/utils/installOutcome.js @@ -8,8 +8,8 @@ * the response stream left to tell them apart. * * INSTALLED and ALREADY_INSTALLED both mean the node holds the app. DECLINED and - * FAILED both mean it does not. BUSY says nothing about the app at all - only about - * the node at that instant. + * FAILED both mean it does not. BUSY and DEFERRED say nothing about the app at all - + * only about the node at that instant. * * Every outcome except BUSY is returned after the install hold is acquired, so each * is a true statement about the app whoever asked for it: concurrent attempts are @@ -39,6 +39,9 @@ const InstallOutcome = Object.freeze({ // The install got part way and cleaned up after itself, so the app is no longer on // this node. The only outcome that justifies a caller acting on the loss. FAILED: 'failed', + // The node's database did not answer, so the attempt stopped where it was and + // tore nothing down. Carries no claim about the app: a later attempt decides. + DEFERRED: 'deferred', }); module.exports = { InstallOutcome }; diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index c8b4ced5c7..72e95c7b91 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -13,6 +13,7 @@ const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileg const dbHelper = require('../../ZelBack/src/services/dbHelper'); const appsRuntimeState = require('../../ZelBack/src/services/appManagement/appsRuntimeState'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); +const { MongoNetworkTimeoutError } = require('mongodb'); const log = require('../../ZelBack/src/lib/log'); const https = require('https'); const syncthingService = require('../../ZelBack/src/services/syncthingService'); @@ -99,6 +100,21 @@ describe('advancedWorkflows tests', () => { expect(globalState.installationInProgress, 'the node is left holding an install that never began').to.be.false; }); + it('stops without tearing anything down when the database does not answer', async () => { + // eslint-disable-next-line global-require + const appUninstaller = require('../../ZelBack/src/services/appLifecycle/appUninstaller'); + sinon.stub(generalService, 'nodeTier').resolves('cumulus'); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + + const result = await advancedWorkflows.softRegisterAppLocally({ name: 'testapp', version: 3 }, false, makeRes()); + + expect(result).to.equal(InstallOutcome.DEFERRED); + expect(removeAppLocally.called, 'a soft registration destroyed the data it exists to keep').to.be.false; + expect(globalState.installationInProgress).to.be.false; + }); + it('does not release a hold it never took', async () => { // Refused because another install owns the node. Lowering the flag here // hands the node to the next caller while that install is still running. @@ -6617,6 +6633,19 @@ describe('advancedWorkflows tests', () => { expect(globalState.softRedeployInProgress).to.be.false; }); + it('leaves the app down with its data when the step after the removal cannot read the database', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); + sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + + await advancedWorkflows.softRedeploy(v3Spec, null); + + expect(removeAppLocally.called, 'the app was uninstalled because the database did not answer').to.be.false; + expect(globalState.softRedeployInProgress).to.be.false; + }); + it('records that a failure after the app is down still force-uninstalls it', async () => { // Pins CURRENT behaviour, not desired behaviour. The gate covers a removal // that never finished; past that point a failure still force-uninstalls @@ -7045,6 +7074,28 @@ describe('advancedWorkflows tests', () => { expect(res.writesAfterEnd).to.be.empty; }); + it('does not uninstall the app when the step after a soft component removal cannot read the database', async () => { + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + const res = streamingRes(); + + await advancedWorkflows.redeployComponentAPI(req, res); + + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.writesAfterEnd).to.be.empty; + }); + + it('does not uninstall the app when the step after a hard component removal cannot read the database', async () => { + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + sinon.stub(appUninstaller, 'hardUninstallComponent').resolves(); + req.params.force = 'true'; + const res = streamingRes(); + + await advancedWorkflows.redeployComponentAPI(req, res); + + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.writesAfterEnd).to.be.empty; + }); + // The gate must not be shut. A teardown that COMPLETED and a failure after it // is the case the forced uninstall exists for, and a suite that proved only // the refusals would pass with the fallback deleted altogether. @@ -7220,6 +7271,26 @@ describe('advancedWorkflows tests', () => { ).to.be.false; }); + it('does not uninstall an app whose redeploy stops on a database that does not answer', async () => { + sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + + await advancedWorkflows.reinstallOldApplications(); + + expect(removeAppLocally.called, 'an app was uninstalled with its data because the database did not answer').to.be.false; + }); + + it('a canary: a redeploy that fails for another reason still uninstalls the app', async () => { + sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').rejects(new Error('Insufficient RAM')); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(removeAppLocally); + }); + it('declines a second pass that starts before the first has announced itself', async () => { const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); @@ -8488,6 +8559,16 @@ describe('a redeploy asks what it may rebuild before it takes anything down', () expect(globalState.hardRedeployInProgress).to.be.false; }); + it('hardRedeploy leaves a running app alone when it fails before the teardown', async () => { + sinon.stub(globalState, 'operationHolding').throws(new Error('connection timed out')); + + await advancedWorkflows.hardRedeploy({ version: 4, name: 'plainapp', nodes: [] }, res); + + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.write.lastCall.args[0]).to.match(/before the app was taken down/); + expect(globalState.hardRedeployInProgress).to.be.false; + }); + it('a canary: hardRedeploy with the gate open reaches the teardown', async () => { await advancedWorkflows.hardRedeploy({ version: 4, name: 'plainapp', nodes: [] }, res); diff --git a/tests/unit/appReconciler.test.js b/tests/unit/appReconciler.test.js index f22d7ee545..5e4112cd83 100644 --- a/tests/unit/appReconciler.test.js +++ b/tests/unit/appReconciler.test.js @@ -5,6 +5,9 @@ const proxyquire = require('proxyquire').noCallThru(); // The real classifier: both the reconciler and the boot sweep classify a mount // fault through this one function, so the mock borrows it rather than restating it. const { classifyVolumeFault } = require('../../ZelBack/src/services/appTamperingDetectionService'); +// The real test of what counts as a database failure, borrowed for the same reason. +const { isDatabaseError } = require('../../ZelBack/src/services/dbHelper'); +const { MongoNetworkTimeoutError } = require('mongodb'); describe('appReconciler tests', () => { let appReconciler; @@ -21,6 +24,7 @@ describe('appReconciler tests', () => { dbHelper: { databaseConnection: () => ({ db: () => ({}) }), findOneInDatabase: sinon.stub().callsFake(async () => localSpec), + isDatabaseError, }, dockerService: { dockerContainerInspect: sinon.stub().resolves({ State: { Running: false, Status: 'exited', ExitCode: 1 } }), @@ -778,6 +782,17 @@ describe('appReconciler tests', () => { expect(stubs.containerHealthMonitor.recreateMissingContainers.calledOnceWith('www_App')).to.be.true; }); + it('does not remove the app when the recreate fails on a database read, and retries', async () => { + stubs.dockerService.dockerContainerInspect.rejects(new Error('Container www_App not found')); + stubs.dockerService.dockerListContainers.resolves([]); // probe: docker is up + stubs.containerHealthMonitor.recreateMissingContainers.rejects(new MongoNetworkTimeoutError('connection timed out')); + + await appReconciler.reconcile('www_App'); + + expect(stubs.appUninstaller.removeAppLocally.called, 'an app was uninstalled because the database did not answer').to.be.false; + expect(stubs.appTamperingDetectionService.recordEvent.calledWithMatch('App', 'recreation_failed')).to.be.false; + }); + it('removes the app locally when recreation fails (docker reachable)', async () => { stubs.dockerService.dockerContainerInspect.rejects(new Error('Container www_App not found')); stubs.dockerService.dockerListContainers.resolves([]); // probe: docker is up diff --git a/tests/unit/fluxService.test.js b/tests/unit/fluxService.test.js index c111fe3d5b..630c5f3185 100644 --- a/tests/unit/fluxService.test.js +++ b/tests/unit/fluxService.test.js @@ -2393,7 +2393,7 @@ describe('fluxService tests', () => { }); }); - it('should return error if appsServiceAppsResources returns error', async () => { + it('should return error if appsServiceAppsResources rejects', async () => { daemonServiceControlRpcsStub.returns({ status: 'success', data: 'info data' }); daemonServiceFluxnodeRpcsStub.returns({ status: 'success', data: 'status data' }); benchmarkServiceGetInfoStub.returns({ status: 'success', data: 'info2 data' }); @@ -2401,7 +2401,7 @@ describe('fluxService tests', () => { benchmarkServiceGetBenchmarksStub.returns({ status: 'success', data: 'benchmarks data' }); appsServiceFluxUsageStub.returns({ status: 'success', data: 'usage data' }); appsServiceListRunningAppsStub.returns({ status: 'success', data: [runningContainer] }); - appsServiceAppsResourcesStub.returns({ status: 'error', data: 'appsResources data' }); + appsServiceAppsResourcesStub.rejects(new Error('appsResources data')); appsServiceGetAppHashesStub.returns({ status: 'success', data: [{ height: 694000, message: true }] }); explorerServiceStub.returns({ status: 'success', data: 'getScannedHeight data' }); fluxCommunicationStub.returns({ status: 'success', data: 'connectedPeersInfo data' }); @@ -2414,7 +2414,7 @@ describe('fluxService tests', () => { sinon.assert.calledOnceWithMatch(res.json, { status: 'error', - data: { code: undefined, name: undefined, message: 'appsResources data' }, + data: { message: 'appsResources data' }, }); }); diff --git a/tests/unit/resourceQueryService.test.js b/tests/unit/resourceQueryService.test.js index 372282417c..2c427290a8 100644 --- a/tests/unit/resourceQueryService.test.js +++ b/tests/unit/resourceQueryService.test.js @@ -472,6 +472,16 @@ describe('resourceQueryService tests', () => { expect(res.json.firstCall.args[0].status).to.equal('error'); }); + it('rejects with the read\'s own error for a caller inside the node', async () => { + const failure = new Error('connection timed out'); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findInDatabase').rejects(failure); + + const outcome = await resourceQueryService.appsResources().catch((err) => err); + + expect(outcome, 'a node that cannot count its apps answered totals').to.equal(failure); + }); + it('should include filesystem overhead for each app/component', async () => { const req = {}; const res = { From c4f9bbc8073f91378c87949a70045ae70aeb72c5 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 10:57:55 +0100 Subject: [PATCH 049/369] fix(db): an app message is resolved only once its registry row is written Promotion stored the message and marked its hash as resolved before writing the app's registry row, and a failed write answered false that nothing read. The hash was never asked about again, so a paid renewal that missed its row left the app at its old expiry, and the expiry pass uninstalled it. - storeAppSpecificationInForce rejects when a read or the write fails, and leaves a row that already holds the message at that height as it is - a promotion marks the hash at each way out, after the row is written; a failure before that leaves the hash for hash sync to resolve again - resolving a message this node already holds writes the registry row of its app, so a promotion stopped part way is completed; the row is the governing message's, as a rebuild from the log writes it - getChainParamsPriceUpdates rejects instead of answering an empty list, which dropped the configured price forks along with the chain's Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 34 ++++---- .../services/appMessaging/messageVerifier.js | 33 +++++--- ZelBack/src/services/utils/chainUtilities.js | 82 +++++++++---------- tests/unit/chainUtilities.test.js | 38 +++++++++ tests/unit/messageVerifier.test.js | 60 ++++++++++++++ tests/unit/registryManager.test.js | 26 ++++++ 6 files changed, 201 insertions(+), 72 deletions(-) create mode 100644 tests/unit/chainUtilities.test.js diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index e2a7b41613..073cf928a2 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -1914,28 +1914,26 @@ async function isGoverningAppMessage(appName, hash) { * force (appMessageChain): the one that governs the name in the message log, and alive at the * current block. Any other message changes nothing: a newer one governs the app, or its term has ended. * A registration and an update are stored alike, so an update renews an app this node's expiry - * pass has already removed, and this node holds what a rebuild from the log would. + * pass has already removed, and this node holds what a rebuild from the log would. A row that + * already holds this message at this height is left as it is. * @param {object} appSpecs the message's specifications, with its hash and height - * @returns {Promise} false when the write failed + * @returns {Promise} Rejects when a read or the write fails: the caller decides what an + * unwritten registry row costs it. */ async function storeAppSpecificationInForce(appSpecs) { return withRegistryWrite(async () => { - try { - if (!await isGoverningAppMessage(appSpecs.name, appSpecs.hash)) return true; - const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); - const currentHeight = syncStatus && syncStatus.data ? syncStatus.data.height : 0; - if (!appMessageChain.isInForce(appSpecs.height, appSpecs.expire, currentHeight)) return true; - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await dbHelper.replaceOneInDatabase(database, globalAppsInformation, { name: appSpecs.name }, appSpecs, { upsert: true }); - fluxEventBus.publish('app:specStored', { name: appSpecs.name, hash: appSpecs.hash }); - await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsLocations, { name: appSpecs.name }); - await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsBroadcasts, { 'data.name': appSpecs.name }); - return true; - } catch (error) { - log.error(`storeAppSpecificationInForce failed for ${appSpecs.name}: ${error.message}`); - return false; - } + if (!await isGoverningAppMessage(appSpecs.name, appSpecs.hash)) return; + const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); + const currentHeight = syncStatus && syncStatus.data ? syncStatus.data.height : 0; + if (!appMessageChain.isInForce(appSpecs.height, appSpecs.expire, currentHeight)) return; + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + const held = await dbHelper.findOneInDatabase(database, globalAppsInformation, { name: appSpecs.name }, { projection: { _id: 0, hash: 1, height: 1 } }); + if (held && held.hash === appSpecs.hash && held.height === appSpecs.height) return; + await dbHelper.replaceOneInDatabase(database, globalAppsInformation, { name: appSpecs.name }, appSpecs, { upsert: true }); + fluxEventBus.publish('app:specStored', { name: appSpecs.name, hash: appSpecs.hash }); + await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsLocations, { name: appSpecs.name }); + await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsBroadcasts, { 'data.name': appSpecs.name }); }); } diff --git a/ZelBack/src/services/appMessaging/messageVerifier.js b/ZelBack/src/services/appMessaging/messageVerifier.js index 37f6fcb2ac..90d080096a 100644 --- a/ZelBack/src/services/appMessaging/messageVerifier.js +++ b/ZelBack/src/services/appMessaging/messageVerifier.js @@ -574,15 +574,24 @@ async function storeGoverningAppSpecifications(appNames) { } /** - * Moves each stored message to this node's record of its payment (moveStoredMessagesToPayments), - * and writes the registry row of each app with a message that moved - * (storeGoverningAppSpecifications). + * Resolves messages this node already holds: moves each to this node's record of its payment + * (moveStoredMessagesToPayments), and writes the registry row of every app they belong to + * (storeGoverningAppSpecifications). A message whose promotion stopped before its registry row + * was written is completed here, so the row is written before the hash is marked as resolved. * @param {Array<{hash: string, txid: string, height: number, value: number}>} records * @returns {Promise} the names of the apps with a message that moved */ async function alignStoredMessagesWithPayments(records) { const moved = await moveStoredMessagesToPayments(records); - await storeGoverningAppSpecifications(moved); + if (records.length === 0) return moved; + const appsDatabase = dbHelper.databaseConnection().db(config.database.appsglobal.database); + const stored = await dbHelper.findInDatabase( + appsDatabase, + globalAppsMessages, + { hash: { $in: records.map((record) => record.hash) } }, + { projection: { _id: 0, 'appSpecifications.name': 1 } }, + ); + await storeGoverningAppSpecifications([...new Set(stored.map((message) => message.appSpecifications.name))]); return moved; } @@ -858,8 +867,10 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { // eslint-disable-next-line global-require const messageStore = require('./messageStore'); await messageStore.storeAppPermanentMessage(permanentAppMessage); - // await update zelapphashes that we already have it stored - await appHashHasMessage(hash); + // The hash is marked as holding its message only once the message's effect on + // the registry is written, at each way out below: a failure before that leaves + // the hash unresolved, and resolving it again writes the registry + // (resolveStoredMessage). const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); const daemonHeight = syncStatus.data.height; @@ -904,12 +915,8 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { updateForSpecifications.hash = permanentAppMessage.hash; updateForSpecifications.height = permanentAppMessage.height; // object of appSpecifications extended for hash and height - const inserted = await storeAppSpecificationInForce(updateForSpecifications); + await storeAppSpecificationInForce(updateForSpecifications); const appName = specifications.name; - if (!inserted) { - globalState.clearPendingUpdates(appName); - return true; - } const pendingUpdates = globalState.getPendingUpdates(appName); if (pendingUpdates.length > 0) { log.info(`Processing ${pendingUpdates.length} pending updates for ${appName}`); @@ -946,6 +953,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { ); if (!messageInfo) { log.error(`Last permanent message for ${specifications.name} not found`); + await appHashHasMessage(hash); return true; } const previousSpecs = messageInfo.appSpecifications; @@ -991,6 +999,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { log.warn(`Apps message ${permanentAppMessage.hash} is underpaid ${valueSat} < ${appPrice * 1e8}`); } } + await appHashHasMessage(hash); return true; } else { // App has expired (actualExpirationHeight <= daemonHeight) @@ -1002,6 +1011,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { // ending the app here would drop a paid app from this node and uninstall it. if (!await isGoverningAppMessage(specifications.name, permanentAppMessage.hash)) { log.info(`App message ${permanentAppMessage.hash} for ${specifications.name} has expired, but does not govern the app. Nothing to clean up.`); + await appHashHasMessage(hash); return true; } log.warn(`App ${specifications.name} has expired (expiration height ${actualExpirationHeight} <= daemon height ${daemonHeight}). Cleaning up stale data.`); @@ -1032,6 +1042,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { await appUninstaller.removeAppLocally(specifications.name, null, true, false, true); } + await appHashHasMessage(hash); return true; } } diff --git a/ZelBack/src/services/utils/chainUtilities.js b/ZelBack/src/services/utils/chainUtilities.js index 903c346091..6ce60d1358 100644 --- a/ZelBack/src/services/utils/chainUtilities.js +++ b/ZelBack/src/services/utils/chainUtilities.js @@ -4,51 +4,47 @@ const log = require('../../lib/log'); /** * To get array of price specifications updates - * @returns {(object|object[])} Returns an array of app objects. + * @returns {Promise} The price forks, oldest first: the configured ones and the chain's. + * Rejects when the chain's cannot be read - a list without them prices against stale forks. */ async function getChainParamsPriceUpdates() { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.chainparams.database); - const chainParamsMessagesCollection = config.database.chainparams.collections.chainMessages; - const query = { version: 'p' }; - const projection = { - projection: { - _id: 0, - }, - }; - const priceMessages = await dbHelper.findInDatabase(database, chainParamsMessagesCollection, query, projection); - const priceForks = []; - config.fluxapps.price.forEach((price) => { - priceForks.push(price); - }); - priceMessages.forEach((data) => { - const splittedMess = data.message.split('_'); - if (splittedMess[4]) { - const dataPoint = { - height: +data.height, - cpu: +splittedMess[1], - ram: +splittedMess[2], - hdd: +splittedMess[3], - minPrice: +splittedMess[4], - port: +splittedMess[5] || 2, - scope: +splittedMess[6] || 6, - staticip: +splittedMess[7] || 3, - }; - priceForks.push(dataPoint); - } - }); - // sort priceForks depending on height - priceForks.sort((a, b) => { - if (a.height > b.height) return 1; - if (a.height < b.height) return -1; - return 0; - }); - return priceForks; - } catch (error) { - log.error(error); - return []; - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.chainparams.database); + const chainParamsMessagesCollection = config.database.chainparams.collections.chainMessages; + const query = { version: 'p' }; + const projection = { + projection: { + _id: 0, + }, + }; + const priceMessages = await dbHelper.findInDatabase(database, chainParamsMessagesCollection, query, projection); + const priceForks = []; + config.fluxapps.price.forEach((price) => { + priceForks.push(price); + }); + priceMessages.forEach((data) => { + const splittedMess = data.message.split('_'); + if (splittedMess[4]) { + const dataPoint = { + height: +data.height, + cpu: +splittedMess[1], + ram: +splittedMess[2], + hdd: +splittedMess[3], + minPrice: +splittedMess[4], + port: +splittedMess[5] || 2, + scope: +splittedMess[6] || 6, + staticip: +splittedMess[7] || 3, + }; + priceForks.push(dataPoint); + } + }); + // sort priceForks depending on height + priceForks.sort((a, b) => { + if (a.height > b.height) return 1; + if (a.height < b.height) return -1; + return 0; + }); + return priceForks; } /** diff --git a/tests/unit/chainUtilities.test.js b/tests/unit/chainUtilities.test.js new file mode 100644 index 0000000000..75f9ef646b --- /dev/null +++ b/tests/unit/chainUtilities.test.js @@ -0,0 +1,38 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('chainUtilities tests', () => { + describe('getChainParamsPriceUpdates', () => { + const configStub = { + database: { chainparams: { database: 'chainparams', collections: { chainMessages: 'chainmessages' } } }, + fluxapps: { price: [{ height: 0, minPrice: 1 }, { height: 1000, minPrice: 2 }] }, + }; + + function load(findInDatabase) { + return proxyquire('../../ZelBack/src/services/utils/chainUtilities', { + config: configStub, + '../dbHelper': { databaseConnection: () => ({ db: () => ({}) }), findInDatabase }, + '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, + }); + } + + it('answers the configured forks and the chain\'s, oldest first', async () => { + const chainUtilities = load(sinon.stub().resolves([{ height: 500, message: 'p_1_2_3_4' }])); + + const forks = await chainUtilities.getChainParamsPriceUpdates(); + + expect(forks.map((fork) => fork.height)).to.deep.equal([0, 500, 1000]); + }); + + // A list without the chain's forks prices every app against stale prices, + // and one without the configured forks too prices nothing. + it('rejects when the chain\'s forks cannot be read', async () => { + const chainUtilities = load(sinon.stub().rejects(new Error('connection timed out'))); + + const outcome = await chainUtilities.getChainParamsPriceUpdates().catch((err) => err); + + expect(outcome, 'a price list was answered without the chain\'s forks').to.be.an('error'); + }); + }); +}); diff --git a/tests/unit/messageVerifier.test.js b/tests/unit/messageVerifier.test.js index baff1f77f0..fa98513414 100644 --- a/tests/unit/messageVerifier.test.js +++ b/tests/unit/messageVerifier.test.js @@ -302,6 +302,66 @@ describe('messageVerifier tests', () => { sinon.assert.calledWith(getPreviousAppSpecsStub, sinon.match({ name: 'testapp' }), 2000000, sinon.match.number); }); + describe('a registry row that could not be written', () => { + const registration = { + type: 'fluxappregister', + version: 1, + appSpecifications: { name: 'testapp', version: 8, owner: 'owner1' }, + hash: 'hash123', + timestamp: Date.now(), + signature: 'sig123', + }; + // appPricePerMonth is stubbed at 1000 FLUX, so a registration is paid in full at that + const paidInFull = { txid: 'txid123', height: 2000000, value: 1000 * 1e8 }; + const markedResolved = () => updateOneInDatabaseStub.getCalls() + .some((call) => call.args[1] === 'appsHashes' && call.args[3]?.$set?.message === true); + + it('leaves the hash unresolved, so the promotion is completed later', async () => { + storeAppSpecificationInForceStub.rejects(new Error('connection timed out')); + const verifier = buildVerifier({ tempMessage: registration, paymentRecord: paidInFull }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(storeAppPermanentMessageStub.called, 'the fixture never reached the promotion').to.be.true; + expect(result).to.equal(false); + expect(markedResolved(), 'the hash was resolved over a registry row that was never written').to.be.false; + }); + + it('marks the hash once the row is written', async () => { + const verifier = buildVerifier({ tempMessage: registration, paymentRecord: paidInFull }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(result).to.equal(true); + expect(markedResolved()).to.be.true; + expect(storeAppSpecificationInForceStub.calledBefore(updateOneInDatabaseStub)).to.be.true; + }); + + it('writes the row of a stored message when its hash is resolved again', async () => { + const stored = { + hash: 'hash123', txid: 'txid123', height: 2000000, valueSat: 200000000, appSpecifications: { name: 'testapp' }, + }; + const verifier = buildVerifier({ + dbHelperOverrides: { + findOneInDatabase: sinon.stub() + .onFirstCall().resolves({ txid: 'txid123', height: 2000000, value: 200000000 }) + .onSecondCall().resolves(stored), // checkAppMessageExistence - held permanently + findInDatabase: sinon.stub().resolves([stored]), + }, + registryManagerOverrides: { + governingAppMessage: sinon.stub().resolves({ appSpecifications: { name: 'testapp' }, hash: 'hash123', height: 2000000 }), + }, + }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(result).to.equal(true); + sinon.assert.calledWithMatch(storeAppSpecificationInForceStub, { name: 'testapp', hash: 'hash123', height: 2000000 }); + expect(markedResolved()).to.be.true; + expect(storeAppSpecificationInForceStub.calledBefore(updateOneInDatabaseStub), 'the hash was resolved before its row was written').to.be.true; + }); + }); + describe('the payment a message is placed by', () => { it('should store the message at this node\'s record of its payment, not the caller\'s', async () => { getPreviousAppSpecsStub.resolves({ owner: 'correctOwner', version: 8 }); diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 9878c2a6e0..4a1332db9c 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1393,6 +1393,32 @@ describe('registryManager tests', () => { expect(await registryManager.isGoverningAppMessage('NoSuchApp', 'second')).to.equal(false); }); + it('rejects when the row cannot be written, never reporting the app stored', async () => { + atHeight(150); + const reg = { type: 'fluxappregister', hash: 'unwritten', height: 100 }; + await logMessages('UnwrittenApp', [reg]); + sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new Error('connection timed out')); + + const failure = await promote('UnwrittenApp', reg).catch((err) => err); + + expect(failure, 'an unwritten row was reported stored').to.be.an('error'); + expect(await stored('UnwrittenApp')).to.equal(null); + }); + + it('leaves a row that already holds the message as it is', async () => { + atHeight(150); + const reg = { type: 'fluxappregister', hash: 'heldhash', height: 100 }; + await logMessages('HeldApp', [reg]); + await promote('HeldApp', reg); + const replace = sinon.spy(dbHelper, 'replaceOneInDatabase'); + const clearErrors = sinon.spy(dbHelper, 'removeDocumentsFromCollection'); + + await promote('HeldApp', reg); + + sinon.assert.notCalled(replace); + expect(clearErrors.called, 'the app\'s install errors were cleared for a write that changed nothing').to.be.false; + }); + it('should store a registration, then the update that follows it', async () => { atHeight(150); const reg = { type: 'fluxappregister', hash: 'oldhash', height: 100 }; From 14f5509c8df854baf114508041f7af077ead3e95 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 10:59:40 +0100 Subject: [PATCH 050/369] fix(db): a block whose chain records could not be written is processed again The block processor moved its scanned height past a block whose writes had failed, so what they recorded was missing for good: - an app payment that could not be recorded: its hash is never asked about, and the app message it paid for never reaches this node; insertTransactions now rejects once its per-payment fallback has run, and the reprocessed block records what is missing - a price fork that could not be written: this node prices apps against stale forks; a soft fork whose write fails on the database fails the block, and one that cannot be read is still skipped Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/explorerService.js | 22 +++++++++- tests/unit/explorerService.test.js | 57 +++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index eb1fad853b..ac265d5d2c 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -394,17 +394,29 @@ async function processInsight(blockDataVerbose, database) { // eslint-disable-next-line no-await-in-loop await processSoftFork(tx.txid, blockDataVerbose.height, message); } catch (error) { + // A message that cannot be read is skipped. One that could not be written is not: + // the block is reprocessed, or this node prices apps without that fork for good. + if (dbHelper.isDatabaseError(error)) throw error; log.error('Error processing soft fork message:', error); - // Continue processing other transactions even if soft fork processing fails } } } } } +/** + * Records the app payments of the blocks processed since the last write. A payment that could + * not be recorded rejects, so the block processor reprocesses its block instead of moving its + * scanned height past it: a payment with no record is never resolved, and the app message it + * paid for never reaches this node. + * @param {object[]} transactions + * @param {object} database + * @returns {Promise} + */ async function insertTransactions(transactions, database) { if (transactions.length > 0) { log.info(`Explorer - insertTransactions - Inserting ${transactions.length} transactions to apps hashes collection`); + let unrecorded = null; try { const options = { ordered: false, @@ -424,10 +436,13 @@ async function insertTransactions(transactions, database) { await dbHelper.updateOneInDatabase(database, appsHashesCollection, query, update, options); } catch (errorTx) { log.error(`Explorer - insertTransactions - Inserting ${transaction.hash} - transaction error - ${errorTx}`); + unrecorded = unrecorded ?? errorTx; } } } + // Reprocessing the block collects its payments again. appsTransactions = []; + if (unrecorded) throw unrecorded; } } @@ -563,8 +578,10 @@ async function processStandard(blockDataVerbose, database) { try { await processSoftFork(tx.txid, blockDataVerbose.height, message); } catch (error) { + // A message that cannot be read is skipped. One that could not be written is not: + // the block is reprocessed, or this node prices apps without that fork for good. + if (dbHelper.isDatabaseError(error)) throw error; log.error('Error processing soft fork message:', error); - // Continue processing other transactions even if soft fork processing fails } } } @@ -2006,6 +2023,7 @@ module.exports = { processStandard, setBlockProccessingCanContinue, getBlockProccessingCanContinue, + insertTransactions, setIsInInitiationOfBP, setZelAppSpecsMigrationDone, restoreDatabaseToBlockheightState, diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index bf6faef7f3..0fba8638bb 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -12,6 +12,7 @@ const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/ const daemonServiceUtils = require('../../ZelBack/src/services/daemonService/daemonServiceUtils'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); +const { MongoNetworkTimeoutError } = require('mongodb'); const globalState = require('../../ZelBack/src/services/utils/globalState'); const log = require('../../ZelBack/src/lib/log'); const { requireMongo } = require('./dbTestHelper'); @@ -2678,6 +2679,62 @@ describe('explorerService tests', () => { }); }); + describe('a chain write that fails', () => { + // A soft fork the foundation sent to itself: a price fork every node applies. + const softForkBlock = () => ({ + height: 1000, + tx: [{ + version: 4, + txid: 'softforktx', + vin: [{ address: config.fluxapps.addressMultisig }], + vout: [{ + valueSat: 0, + scriptPubKey: { + addresses: [config.fluxapps.addressMultisig], + asm: `OP_RETURN ${Buffer.from('p_1_2_3_4').toString('hex')}`, + }, + }], + }], + }); + + beforeEach(() => { + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(log, 'error'); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('fails the block when its soft fork cannot be written', async () => { + sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + + const outcome = await explorerService.processInsight(softForkBlock(), {}).catch((err) => err); + + expect(outcome, 'the block moved on without its price fork').to.be.instanceOf(MongoNetworkTimeoutError); + }); + + it('a canary: a soft fork that fails for another reason is skipped, as before', async () => { + const updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new Error('unreadable message')); + + await explorerService.processInsight(softForkBlock(), {}); + + sinon.assert.called(updateOne); + }); + + it('fails the block when one of its app payments cannot be recorded', async () => { + sinon.stub(dbHelper, 'insertManyToDatabase').rejects(new Error('batch failed')); + const upsert = sinon.stub(dbHelper, 'updateOneInDatabase'); + upsert.onFirstCall().resolves(); + upsert.onSecondCall().rejects(new MongoNetworkTimeoutError('connection timed out')); + + const outcome = await explorerService.insertTransactions([{ hash: 'a', height: 1 }, { hash: 'b', height: 1 }], {}).catch((err) => err); + + expect(outcome, 'the block moved on with a payment unrecorded').to.be.instanceOf(MongoNetworkTimeoutError); + sinon.assert.calledTwice(upsert); + }); + }); + describe('explorer reset requests', () => { let res; let clock; From c7e2d118b70684a15d9b021d93792e938d726746 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:01:19 +0100 Subject: [PATCH 051/369] fix(db): a folder whose app locations cannot be read is left as it is The syncthing pass read an app's locations through a reader that answered no locations when the read failed. The folder was then written shared with this node alone, and the device sweep removed every peer it had: a primary stopped replicating to its standbys for as long as the read failed. A folder whose locations cannot be read is held out of the pass, as an unmounted volume is: its configuration stands, and the device sweep stands down while anything is held out. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/syncthingMonitor.js | 14 ++++++++++- tests/unit/syncthingMonitor.test.js | 25 ++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 8052b1858b..a4948fce42 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -289,6 +289,7 @@ async function processContainerData(params) { newFoldersConfiguration, liveness, electedFolderIds = [], + unsafeFolderIds, } = params; const containersData = containerData.split('|'); @@ -332,7 +333,17 @@ async function processContainerData(params) { } // Get and process app locations - let locations = await appLocation(installedAppName); + let locations; + try { + locations = await readAppLocation(installedAppName); + } catch (error) { + // Without its locations the folder would be shared with no peer, and the device + // sweep would remove every peer it has: it is held out of this pass instead. + log.warn(`processContainerData - ${appId} locations could not be read (${error.message}); leaving the folder as it is this cycle`); + fluxEventBus.count('syncthing:folderPass', appId, 'locationsUnread'); + unsafeFolderIds.add(appId); + return; + } locations = sortAndFilterLocations(locations, localSocketAddr); // Build device configuration (parallelized internally) @@ -806,6 +817,7 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { folderIds, foldersConfiguration, newFoldersConfiguration, + unsafeFolderIds, }; // Process all installed apps diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index e0c30efc18..7df849ac91 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -228,6 +228,9 @@ describe('syncthingMonitor tests', () => { peerFolderLivenessMock.createPeerFolderLiveness.resetHistory(); dbHelperMock.databaseConnection.reset(); dbHelperMock.findInDatabase.reset(); + // An app's location records, read for every folder the pass configures: none. + dbHelperMock.databaseConnection.returns({ db: () => ({}) }); + dbHelperMock.findInDatabase.resolves([]); // Default stub behaviors // The node's own identity, which a real pass always has: mount safety is @@ -1578,7 +1581,9 @@ describe('syncthingMonitor tests', () => { // could not be read"; the read it is handed must not collapse the two. it('hands the state machine a location read that fails when the records cannot be read', async () => { writesAFolder(); - dbHelperMock.databaseConnection.throws(new Error('database unavailable')); + // read once for the folder's devices, then unreadable for the state machine + dbHelperMock.findInDatabase.onFirstCall().resolves([]); + dbHelperMock.findInDatabase.rejects(new Error('database unavailable')); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); @@ -1874,6 +1879,24 @@ describe('syncthingMonitor tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigFolders, 'delete', undefined, 'testapp'); }); + it('leaves the folder and its peers alone when the app\'s locations cannot be read', async () => { + // Read as "no locations", the folder would be pushed shared with this node + // alone and every peer device it had removed - a primary would stop + // replicating to its standbys for as long as the read failed. + primaryMountSyncs(); + mockInstalledAppsFn.resolves({ status: 'success', data: [syncingApp] }); + dbHelperMock.findInDatabase.rejects(new Error('connection timed out')); + syncthingServiceMock.getConfigFolders.resolves([{ id: 'testapp', type: 'sendreceive', devices: [{ deviceID: 'DEVICE-ID' }, { deviceID: 'PEER-ID' }] }]); + syncthingServiceMock.getConfigDevices.resolves([{ deviceID: 'PEER-ID', name: '10.0.0.2:16127' }]); + syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + + await runOnePass(); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-ID'); + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigFolders, 'delete', undefined, 'testapp'); + expect(syncthingFolderStateMachineMock.manageFolderSyncState.called, 'the folder was configured without its locations').to.be.false; + }); + it('keeps the folder of a component the state machine defers on first encounter', async () => { // the first-encounter deferral is a "look again next pass", not a verdict // that the folder should not exist From c9f97cc399e105825711b49a617bd07e110f5c8c Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:04:46 +0100 Subject: [PATCH 052/369] fix(db): boot decides nothing about an app on a read that failed - a location record that cannot be read is neither valid nor expired: the boot check neither starts the app, which would run it beside the instance placed in its stead, nor removes it, and the reconciler decides it later; appHasValidLocationOnNode answers null - a boot context that cannot be read is reported as unreadable rather than as a first boot: its downtime is unknown, so no app is removed wholesale, and each app's own location decides - the installed apps read at boot rejects instead of answering none; the reconciler's sweep at the first announcement reconciles them - the reason the previous process stopped is cleared by the first heartbeat that lands rather than by one write whose failure was logged: left standing, a crash of this process reads as that clean stop at the next boot Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 30 +++++++++++-------- .../appMessaging/appSyncOrchestrator.js | 22 +++++++------- ZelBack/src/services/utils/appUtilities.js | 9 +++++- tests/unit/appStartupManager.test.js | 27 +++++++++++++---- tests/unit/appSyncOrchestrator.test.js | 25 +++++++++++++--- 5 files changed, 77 insertions(+), 36 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index b87e53151f..1ae777d88f 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -52,19 +52,14 @@ async function awaitWithin(work, timeoutMs, reason) { * @returns {Promise} Array of installed app specifications */ async function getInstalledAppsFromDb() { - try { - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const appsQuery = {}; - const appsProjection = { - projection: { _id: 0 }, - }; - const apps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - return apps || []; - } catch (error) { - log.error(`appStartupManager - Error getting installed apps: ${error.message}`); - return []; - } + const dbopen = dbHelper.databaseConnection(); + const appsDatabase = dbopen.db(config.database.appslocal.database); + const appsQuery = {}; + const appsProjection = { + projection: { _id: 0 }, + }; + const apps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); + return apps || []; } /** @@ -227,6 +222,13 @@ async function reconcileAppsOnBoot() { if (localSocketAddr) { // eslint-disable-next-line no-await-in-loop const hasValidLocation = await appHasValidLocationOnNode(appName, localSocketAddr); + if (hasValidLocation === null) { + // Neither started nor removed on an unread location: the reconciler decides later. + log.warn(`appStartupManager - App ${appName}: its location record could not be read, leaving it as it is`); + results.appsFailed.push({ app: appName, error: 'location record unreadable' }); + // eslint-disable-next-line no-continue + continue; + } if (!hasValidLocation) { log.warn(`appStartupManager - App ${appName} no longer has a valid location record for this node (${localSocketAddr}), removing locally`); try { @@ -306,6 +308,8 @@ async function manageAppsOnBoot(bootContext) { try { if (bootContext.firstBoot) { log.info('appStartupManager - First boot (no heartbeat history), waiting for sync'); + } else if (bootContext.unreadable) { + log.warn('appStartupManager - Boot context unreadable: whether locations expired is decided app by app'); } else { const locationsExpired = (bootContext.cleanShutdown && bootContext.downtimeMs > SIGTERM_EXPIRY_MS) || bootContext.downtimeMs > RUNNING_EXPIRY_MS; diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index c3f29f08da..7d0088a098 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -1124,22 +1124,19 @@ class AppSyncOrchestrator { log.info(`Boot context: machineRebooted=${machineRebooted} downtime=${Math.round(downtimeMs / 1000)}s cleanShutdown=${cleanShutdown} firstBoot=${!heartbeat}`); return ctx; } catch (error) { + // Unread is neither a first boot nor a long absence: how long the node was down is + // unknown, so whether its locations expired is decided app by app. log.error(`Failed to read boot context: ${error.message}`); - return { machineRebooted: true, downtimeMs: Infinity, cleanShutdown: false, currentBootId: null, firstBoot: true }; - } - } - - async #clearShutdownReason() { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.local.database); - await dbHelper.findOneAndUpdateInDatabase(database, startupCollection, { _id: 'heartbeat' }, { $unset: { shutdownReason: '' } }); - } catch (error) { - log.error(`Failed to clear shutdown reason: ${error.message}`); + return { + unreadable: true, machineRebooted: true, downtimeMs: null, cleanShutdown: false, currentBootId: null, firstBoot: false, + }; } } #startHeartbeat() { + // The reason the previous process stopped is its own: cleared by the first heartbeat + // that lands, so a crash of this one never reads as that stop. + let shutdownReasonCleared = false; const writeHeartbeat = async () => { try { const db = dbHelper.databaseConnection(); @@ -1148,12 +1145,13 @@ class AppSyncOrchestrator { if (this.#bootContext?.currentBootId) { update.$set.machineBootId = this.#bootContext.currentBootId; } + if (!shutdownReasonCleared) update.$unset = { shutdownReason: '' }; await dbHelper.findOneAndUpdateInDatabase(database, startupCollection, { _id: 'heartbeat' }, update, { upsert: true }); + shutdownReasonCleared = true; } catch (error) { log.error(`Heartbeat write failed: ${error.message}`); } }; - this.#clearShutdownReason(); writeHeartbeat(); this.#heartbeatInterval = setInterval(writeHeartbeat, config.system.heartbeatIntervalMs ?? 30000); } diff --git a/ZelBack/src/services/utils/appUtilities.js b/ZelBack/src/services/utils/appUtilities.js index 318ad2a933..edd75bf6f0 100644 --- a/ZelBack/src/services/utils/appUtilities.js +++ b/ZelBack/src/services/utils/appUtilities.js @@ -1198,6 +1198,13 @@ function parseContainerName(containerName) { }; } +/** + * Whether this node holds an unexpired location record for an app. + * @param {string} appName + * @param {string} localSocketAddr + * @returns {Promise} null when the records cannot be read: an unread record is + * neither a valid nor an expired one. + */ async function appHasValidLocationOnNode(appName, localSocketAddr) { try { const db = dbHelper.databaseConnection(); @@ -1215,7 +1222,7 @@ async function appHasValidLocationOnNode(appName, localSocketAddr) { }); } catch (error) { log.error(`Error checking app location for ${appName}: ${error.message}`); - return true; + return null; } } diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index e53acd5e5b..3c2afb9c75 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -160,12 +160,12 @@ describe('appStartupManager tests', () => { expect(result).to.equal(true); }); - it('should return true on database error (fail-safe)', async () => { + it('answers unknown when the records cannot be read, neither valid nor expired', async () => { dbHelperStub.findInDatabase.rejects(new Error('DB connection lost')); const result = await appUtilities.appHasValidLocationOnNode('myApp', '10.0.0.1:16127'); - expect(result).to.equal(true); + expect(result).to.equal(null); }); it('should query with correct app name and IP', async () => { @@ -332,19 +332,20 @@ describe('appStartupManager tests', () => { expect(results.appsFailed[0].error).to.equal('Remove failed'); }); - it('should still start app when location DB check errors (fail-safe)', async () => { + it('neither starts nor removes an app whose location record cannot be read', async () => { + // Started, an app whose location expired runs beside the instance placed in + // its stead; removed, a valid one is lost. The reconciler decides it later. dockerServiceStub.dockerListContainers.resolves([ { Names: ['/fluxAppA'], State: 'exited' }, ]); dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); - - // Location check throws error - appHasValidLocationOnNode returns true (fail-safe) dbHelperStub.findInDatabase.onSecondCall().rejects(new Error('DB error')); const results = await appStartupManager.reconcileAppsOnBoot(); - expect(results.appsStarted).to.deep.equal(['AppA']); + expect(results.appsStarted).to.deep.equal([]); expect(results.appsRemoved).to.deep.equal([]); + expect(results.appsFailed.map((failure) => failure.app)).to.deep.equal(['AppA']); }); it('should check location after g: syncthing check', async () => { @@ -515,6 +516,20 @@ describe('appStartupManager tests', () => { expect(appUninstallerStub.removeAppLocally.called).to.be.false; }); + it('removes nothing wholesale when the boot context could not be read', async () => { + const bootContext = { + unreadable: true, machineRebooted: true, downtimeMs: null, cleanShutdown: false, firstBoot: false, + }; + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); + dockerServiceStub.dockerListContainers.resolves([]); + dbHelperStub.findInDatabase.resolves([]); + + await appStartupManager.manageAppsOnBoot(bootContext); + + expect(appUninstallerStub.removeAppLocally.called, 'every app was removed on an unread downtime').to.be.false; + expect(globalStateStub.waitForDbReady.calledOnce, 'the apps are reconciled one by one instead').to.be.true; + }); + it('should remove all apps when clean shutdown and downtime > SIGTERM_EXPIRY', async () => { const bootContext = { machineRebooted: true, downtimeMs: 500000, cleanShutdown: true, diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index 20fe7d796d..f1de286633 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -2372,15 +2372,15 @@ describe('AppSyncOrchestrator', () => { expect(ctx.downtimeMs).to.be.within(299000, 301000); }); - it('should return safe defaults on error', async () => { + it('reports an unread context as unreadable, neither a first boot nor a long absence', async () => { dbHelperStub.findOneInDatabase.rejects(new Error('DB down')); const ctx = await AppSyncOrchestrator.readBootContext(); - expect(ctx.machineRebooted).to.be.true; - expect(ctx.downtimeMs).to.equal(Infinity); + expect(ctx.unreadable).to.be.true; + expect(ctx.firstBoot).to.be.false; + expect(ctx.downtimeMs, 'an unknown downtime read as expired removes every app').to.equal(null); expect(ctx.cleanShutdown).to.be.false; - expect(ctx.firstBoot).to.be.true; }); }); @@ -2415,6 +2415,23 @@ describe('AppSyncOrchestrator', () => { await orchestrator.stop(); }); + it('clears the previous shutdown reason with each heartbeat until one lands', async () => { + const heartbeats = () => findOneAndUpdateStub.getCalls() + .filter((c) => c.args[2]?._id === 'heartbeat' && c.args[3]?.$set && 'lastAlive' in c.args[3].$set); + findOneAndUpdateStub.onFirstCall().rejects(new Error('connection timed out')); + const orchestrator = makeOrchestrator(); + await orchestrator.start(defaultBootContext); + // tests/unit/globalconfig: system.heartbeatIntervalMs + await clock.tickAsync(30000); + await clock.tickAsync(30000); + + const [failed, landed, after] = heartbeats(); + expect(failed.args[3].$unset, 'the first heartbeat carries the clear').to.deep.equal({ shutdownReason: '' }); + expect(landed.args[3].$unset, 'a failed clear is carried by the next heartbeat').to.deep.equal({ shutdownReason: '' }); + expect(after.args[3].$unset, 'cleared once it landed, never again').to.equal(undefined); + await orchestrator.stop(); + }); + it('should store boot context and expose via getter', async () => { const orchestrator = makeOrchestrator(); await orchestrator.start(defaultBootContext); From 88ffa71a2be0566c34cd07792c600757c522460f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:08:14 +0100 Subject: [PATCH 053/369] fix(db): an app state record that was not written is not served or credited This node serves its app state event log, its signed installing claims and its location rows to the peers that sync from it. A write to any of them that failed was logged and taken as done. - the state event handlers (apprunning, sigterm, appremoved, evicted, ipchanged) reject when the record is not written, so the gossip handler neither updates locations nor relays the message as recorded; the same message arriving from another peer records it again - a signed installing or installing-error claim is awaited, and one that is not recorded is not relayed - every batch store of a sync response reports writeFailed, and a response whose writes failed is not credited as a completed survey: the peer has not answered, and another is asked - a failed prune of stale location rows rejects instead of being logged Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMessaging/messageStore.js | 207 +++++++++--------- ZelBack/src/services/fluxCommunication.js | 31 ++- tests/unit/fluxCommunication.test.js | 55 +++++ tests/unit/messageStore.test.js | 61 ++++++ 4 files changed, 236 insertions(+), 118 deletions(-) diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index fb59298604..d75af73507 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -766,7 +766,7 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); - const { stored } = await storeBatchAppRunningEvents(verifiedBroadcasts); + const { stored, writeFailed: eventsFailed } = await storeBatchAppRunningEvents(verifiedBroadcasts); // One operation is built per app per broadcast and each carries a merge // pipeline, so a sync response of a few thousand broadcasts expands into tens @@ -774,7 +774,7 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { // the driver's encoding of it small; the operations are independent of one // another, which is what makes the unordered write safe to split. const locationOps = []; - let writeFailed = false; + let writeFailed = eventsFailed; const flushLocationOps = async () => { if (!locationOps.length) return; const batch = locationOps.splice(0, locationOps.length); @@ -858,8 +858,7 @@ async function pruneAppRunningLocations(newestByIp) { const flush = async () => { if (!ops.length) return; const batch = ops.splice(0, ops.length); - await database.collection(globalAppsLocations).bulkWrite(batch, { ordered: false }) - .catch((err) => log.error(`pruneAppRunningLocations: ${err.message}`)); + await database.collection(globalAppsLocations).bulkWrite(batch, { ordered: false }); }; const newest = [...newestByIp]; @@ -883,76 +882,64 @@ async function pruneAppRunningLocations(newestByIp) { // --- Event Log Functions --- async function handleAppRunningEvent({ signedBroadcast }) { - try { - const { data } = signedBroadcast; - if (!data || !data.ip || !data.broadcastedAt) return; - if (data.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) return; - - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); + const { data } = signedBroadcast; + if (!data || !data.ip || !data.broadcastedAt) return; + if (data.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) return; - if (data.version === 2 && (!data.apps || data.apps.length === 0)) { - const existing = await database.collection(globalAppStateEvents).findOne({ ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING }); - if (!existing) return; - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); - const dedupKey = data.apps ? 'v2' : `v1:${data.name}`; - const envelope = { version: signedBroadcast.version, timestamp: signedBroadcast.timestamp, pubKey: signedBroadcast.pubKey, signature: signedBroadcast.signature }; - - await database.collection(globalAppStateEvents).updateOne( - { ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey }, - buildConditionalUpsert(data.broadcastedAt, { - ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey, - broadcastedAt: new Date(data.broadcastedAt), - expireAt: new Date(data.broadcastedAt + RUNNING_EXPIRY_MS), - data, envelope, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(apprunning): ${err.message}`); + if (data.version === 2 && (!data.apps || data.apps.length === 0)) { + const existing = await database.collection(globalAppStateEvents).findOne({ ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING }); + if (!existing) return; } + + const dedupKey = data.apps ? 'v2' : `v1:${data.name}`; + const envelope = { version: signedBroadcast.version, timestamp: signedBroadcast.timestamp, pubKey: signedBroadcast.pubKey, signature: signedBroadcast.signature }; + + await database.collection(globalAppStateEvents).updateOne( + { ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey }, + buildConditionalUpsert(data.broadcastedAt, { + ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey, + broadcastedAt: new Date(data.broadcastedAt), + expireAt: new Date(data.broadcastedAt + RUNNING_EXPIRY_MS), + data, envelope, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } async function handleSigtermEvent({ message, envelope }) { if (!message || !message.ip || !message.broadcastedAt) return; - try { - const { ip } = message; - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm' }, - buildConditionalUpsert(message.broadcastedAt, { - ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm', - broadcastedAt: new Date(message.broadcastedAt), - expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), - envelope, data: message, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(sigterm): ${err.message}`); - } + const { ip } = message; + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm' }, + buildConditionalUpsert(message.broadcastedAt, { + ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm', + broadcastedAt: new Date(message.broadcastedAt), + expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), + envelope, data: message, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } async function handleAppRemovedStateEvent({ message, envelope }) { if (!message || !message.ip || !message.appName || !message.broadcastedAt) return; - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}` }, - buildConditionalUpsert(message.broadcastedAt, { - ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}`, - broadcastedAt: new Date(message.broadcastedAt), - expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), - envelope, data: message, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(appremoved): ${err.message}`); - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}` }, + buildConditionalUpsert(message.broadcastedAt, { + ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}`, + broadcastedAt: new Date(message.broadcastedAt), + expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), + envelope, data: message, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } /** @@ -984,20 +971,16 @@ function evictionTime(evictedAt) { async function handleEvictedEvent({ ip, evictedAt }) { if (!ip) return; const at = evictionTime(evictedAt); - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip, type: APP_STATE_EVENT_TYPES.EVICTED, dedupKey: 'evicted' }, - { - $set: { receivedAt: new Date() }, - $max: { createdAt: new Date(at), expireAt: new Date(at + EVICTED_EXPIRY_MS) }, - }, - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(evicted): ${err.message}`); - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip, type: APP_STATE_EVENT_TYPES.EVICTED, dedupKey: 'evicted' }, + { + $set: { receivedAt: new Date() }, + $max: { createdAt: new Date(at), expireAt: new Date(at + EVICTED_EXPIRY_MS) }, + }, + { upsert: true }, + ); } /** @@ -1027,22 +1010,18 @@ async function applyEviction(ip, evictedAt) { async function handleIPChangedEvent({ message, envelope }) { if (!message || !message.oldIP || !message.newIP || !message.broadcastedAt) return; - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged' }, - buildConditionalUpsert(message.broadcastedAt, { - ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged', - broadcastedAt: new Date(message.broadcastedAt), - expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), - data: message, envelope: envelope ?? null, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(ipchanged): ${err.message}`); - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged' }, + buildConditionalUpsert(message.broadcastedAt, { + ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged', + broadcastedAt: new Date(message.broadcastedAt), + expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), + data: message, envelope: envelope ?? null, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } function storeAppStateEvent(type, payload) { @@ -1087,12 +1066,16 @@ async function storeBatchAppRunningEvents(verifiedBroadcasts) { } + let writeFailed = false; if (ops.length > 0) { await database.collection(globalAppStateEvents).bulkWrite(ops, { ordered: false }) - .catch((err) => log.error(`storeBatchAppRunningEvents: ${err.message}`)); + .catch((err) => { + writeFailed = true; + log.error(`storeBatchAppRunningEvents: ${err.message}`); + }); } - return { stored: ops.length }; + return { stored: ops.length, writeFailed }; } function storeSignedAppInstallingBroadcast(signedBroadcast) { @@ -1116,11 +1099,16 @@ function storeSignedAppInstallingBroadcast(signedBroadcast) { { 'data.name': data.name, 'data.ip': data.ip }, { $set: doc }, { upsert: true }, - ).catch((err) => log.error(`storeSignedAppInstallingBroadcast: ${err.message}`)); + ); } async function storeBatchAppInstallingMessages(verifiedBroadcasts) { - if (verifiedBroadcasts.length === 0) return { stored: 0 }; + if (verifiedBroadcasts.length === 0) return { stored: 0, writeFailed: false }; + let writeFailed = false; + const failed = (what, err) => { + writeFailed = true; + log.error(`storeBatchAppInstallingMessages ${what}: ${err.message}`); + }; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -1189,17 +1177,17 @@ async function storeBatchAppInstallingMessages(verifiedBroadcasts) { if (signedOps.length > 0) { await database.collection(appsInstallingBroadcasts).bulkWrite(signedOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingMessages signed: ${err.message}`)); + .catch((err) => failed('signed', err)); } if (locationOps.length > 0) { await database.collection(globalAppsInstallingLocations).bulkWrite(locationOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingMessages locations: ${err.message}`)); + .catch((err) => failed('locations', err)); } // after the claims, so a claim and its withdrawal arriving in one batch settle // in the order they were sent rather than the order they were read if (withdrawalOps.length > 0) { await database.collection(globalAppsInstallingLocations).bulkWrite(withdrawalOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingMessages withdrawals: ${err.message}`)); + .catch((err) => failed('withdrawals', err)); await database.collection(appsInstallingBroadcasts).bulkWrite( // The location's own guard, carried over rather than restated: the // broadcast is what proves the location, so deleting one without the @@ -1215,9 +1203,9 @@ async function storeBatchAppInstallingMessages(verifiedBroadcasts) { }, })), { ordered: false }, - ).catch((err) => log.error(`storeBatchAppInstallingMessages withdrawal broadcasts: ${err.message}`)); + ).catch((err) => failed('withdrawal broadcasts', err)); } - return { stored: signedOps.length + withdrawalOps.length }; + return { stored: signedOps.length + withdrawalOps.length, writeFailed }; } function storeSignedAppInstallingErrorBroadcast(signedBroadcast) { @@ -1241,11 +1229,16 @@ function storeSignedAppInstallingErrorBroadcast(signedBroadcast) { { 'data.name': data.name, 'data.hash': data.hash, 'data.ip': data.ip }, { $set: doc }, { upsert: true }, - ).catch((err) => log.error(`storeSignedAppInstallingErrorBroadcast: ${err.message}`)); + ); } async function storeBatchAppInstallingErrorMessages(verifiedBroadcasts) { - if (verifiedBroadcasts.length === 0) return { stored: 0 }; + if (verifiedBroadcasts.length === 0) return { stored: 0, writeFailed: false }; + let writeFailed = false; + const failed = (what, err) => { + writeFailed = true; + log.error(`storeBatchAppInstallingErrorMessages ${what}: ${err.message}`); + }; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -1298,13 +1291,13 @@ async function storeBatchAppInstallingErrorMessages(verifiedBroadcasts) { if (signedOps.length > 0) { await database.collection(globalAppsInstallingErrorsBroadcasts).bulkWrite(signedOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingErrorMessages signed: ${err.message}`)); + .catch((err) => failed('signed', err)); } if (locationOps.length > 0) { await database.collection(globalAppsInstallingErrorsLocations).bulkWrite(locationOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingErrorMessages locations: ${err.message}`)); + .catch((err) => failed('locations', err)); } - return { stored: signedOps.length }; + return { stored: signedOps.length, writeFailed }; } module.exports = { diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 305fd43fcd..f2ee98cc4d 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -238,6 +238,7 @@ async function handleAppRunningSyncResponse(message, peerSocket) { }); if (locationWriteFailed) { + // Not credited either: what this peer sent is not all recorded, so it has not answered. log.warn('handleAppRunningSyncResponse - skipping location pruning, a location write failed'); } else { await messageStore.pruneAppRunningLocations(newestByIp); @@ -261,7 +262,7 @@ async function handleAppRunningSyncResponse(message, peerSocket) { } } - if (done) { + if (done && !locationWriteFailed) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apprunning', peerKey); log.info('handleAppRunningSyncResponse - Sync complete'); } @@ -284,14 +285,17 @@ async function handleAppInstallingSyncResponse(message, peerSocket) { } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppInstallingSyncResponse - Received ${messages.length} broadcasts from ${peerKey} (done: ${!!done})`); + let writeFailed = false; await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { const verified = await batchVerifyBroadcasts(slice, 'handleAppInstallingSyncResponse'); if (verified.length === 0) return; - const { stored } = await messageStore.storeBatchAppInstallingMessages(verified); - log.info(`handleAppInstallingSyncResponse - Stored ${stored} of ${verified.length} verified broadcasts`); - fluxEventBus.publish('sync:chunkVerified', { syncType: 'appinstalling', peer: peerKey, verified: verified.length, stored }); + const stored = await messageStore.storeBatchAppInstallingMessages(verified); + if (stored.writeFailed) writeFailed = true; + log.info(`handleAppInstallingSyncResponse - Stored ${stored.stored} of ${verified.length} verified broadcasts`); + fluxEventBus.publish('sync:chunkVerified', { syncType: 'appinstalling', peer: peerKey, verified: verified.length, stored: stored.stored }); }); - if (done) { + // A response whose writes failed is not all recorded: this peer has not answered. + if (done && !writeFailed) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'appinstalling', peerKey); log.info('handleAppInstallingSyncResponse - Sync complete'); } @@ -314,14 +318,17 @@ async function handleAppInstallingErrorsSyncResponse(message, peerSocket) { } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppInstallingErrorsSyncResponse - Received ${messages.length} broadcasts from ${peerKey} (done: ${!!done})`); + let writeFailed = false; await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { const verified = await batchVerifyBroadcasts(slice, 'handleAppInstallingErrorsSyncResponse'); if (verified.length === 0) return; - const { stored } = await messageStore.storeBatchAppInstallingErrorMessages(verified); - log.info(`handleAppInstallingErrorsSyncResponse - Stored ${stored} of ${verified.length} verified broadcasts`); - fluxEventBus.publish('sync:chunkVerified', { syncType: 'apperrors', peer: peerKey, verified: verified.length, stored }); + const stored = await messageStore.storeBatchAppInstallingErrorMessages(verified); + if (stored.writeFailed) writeFailed = true; + log.info(`handleAppInstallingErrorsSyncResponse - Stored ${stored.stored} of ${verified.length} verified broadcasts`); + fluxEventBus.publish('sync:chunkVerified', { syncType: 'apperrors', peer: peerKey, verified: verified.length, stored: stored.stored }); }); - if (done) { + // A response whose writes failed is not all recorded: this peer has not answered. + if (done && !writeFailed) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apperrors', peerKey); log.info('handleAppInstallingErrorsSyncResponse - Sync complete'); } @@ -428,7 +435,9 @@ async function handleAppInstallingMessage(message, fromIP, port) { withdrawn: message.data.withdrawn === true, }); } - messageStore.storeSignedAppInstallingBroadcast(message); + // The signed claim is what this node serves to peers syncing from it: one that is + // not recorded is not relayed as though it were. + await messageStore.storeSignedAppInstallingBroadcast(message); const currentTimeStamp = Date.now(); const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(message, currentTimeStamp); if (rebroadcastToPeers === true && timestampOK) { @@ -451,7 +460,7 @@ async function handleAppInstallingErrorMessage(message, fromIP, port) { if (rebroadcastToPeers === true) { fluxEventBus.publish('network:appinstallingerror', { ip: message.data.ip, name: message.data.name }); } - messageStore.storeSignedAppInstallingErrorBroadcast(message); + await messageStore.storeSignedAppInstallingErrorBroadcast(message); const currentTimeStamp = Date.now(); const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(message, currentTimeStamp); if (rebroadcastToPeers === true && timestampOK) { diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index 6d4f592a36..3e8a5d558a 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -2371,6 +2371,61 @@ describe('fluxCommunication tests', () => { // "nothing after this". A chunk taking its place in the queue only once its // envelope check came back put the order in the hands of whichever check // finished first. + // A response whose writes failed is not all recorded here: the peer has not + // answered, so another is asked. Credited, the node goes authoritative on a + // partial view and serves it to its own peers as complete. + describe('a response whose writes failed', () => { + const broadcast = () => ({ + version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig', + data: { ip: '1.2.3.4:16127', name: 'a', broadcastedAt: Date.now() }, + }); + + beforeEach(() => { + sinon.stub(fluxCommunicationUtils, 'verifyBroadcasts') + .callsFake(async (items) => items.map(() => ({ result: fluxCommunicationUtils.VerifyResult.OK }))); + }); + + it('is not counted as a completed appinstalling survey', async () => { + sinon.stub(messageStore, 'storeBatchAppInstallingMessages').resolves({ stored: 1, writeFailed: true }); + + await fluxCommunication.handleAppInstallingSyncResponse({ data: { type: 'fluxappinstallingsync', messages: [broadcast()], done: true } }, PEER_SOCKET); + + expect(messageStore.storeBatchAppInstallingMessages.called, 'the fixture never reached the store').to.be.true; + expect(completions).to.deep.equal([]); + }); + + it('is not counted as a completed apperrors survey', async () => { + sinon.stub(messageStore, 'storeBatchAppInstallingErrorMessages').resolves({ stored: 1, writeFailed: true }); + + await fluxCommunication.handleAppInstallingErrorsSyncResponse({ data: { type: 'fluxappinstallingerrorssync', messages: [broadcast()], done: true } }, PEER_SOCKET); + + expect(messageStore.storeBatchAppInstallingErrorMessages.called, 'the fixture never reached the store').to.be.true; + expect(completions).to.deep.equal([]); + }); + + it('is not counted as a completed apprunning survey', async () => { + sinon.stub(messageStore, 'storeBatchAppRunningMessages').resolves({ stored: 1, writeFailed: true }); + const event = { + type: 'apprunning', + envelope: { version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig' }, + data: { version: 2, ip: '1.2.3.4:16127', broadcastedAt: Date.now(), apps: [{ name: 'a' }] }, + }; + + await fluxCommunication.handleAppRunningSyncResponse({ data: { type: 'fluxapprunningsync', messages: [event], done: true } }, PEER_SOCKET); + + expect(messageStore.storeBatchAppRunningMessages.called, 'the fixture never reached the store').to.be.true; + expect(completions).to.deep.equal([]); + }); + + it('a canary: the same response written in full is counted', async () => { + sinon.stub(messageStore, 'storeBatchAppInstallingMessages').resolves({ stored: 1, writeFailed: false }); + + await fluxCommunication.handleAppInstallingSyncResponse({ data: { type: 'fluxappinstallingsync', messages: [broadcast()], done: true } }, PEER_SOCKET); + + expect(completions).to.deep.equal([{ syncType: 'appinstalling', peerKey: PEER }]); + }); + }); + describe('a chunk takes its place in the queue when it arrives', () => { const chunkFor = (marker) => ({ data: { type: 'fluxapprunningsync', messages: [{ marker }], done: false }, diff --git a/tests/unit/messageStore.test.js b/tests/unit/messageStore.test.js index bec94d0cf0..a038555281 100644 --- a/tests/unit/messageStore.test.js +++ b/tests/unit/messageStore.test.js @@ -1099,4 +1099,65 @@ describe('messageStore tests', () => { expect($set.expireAt.getTime()).to.equal(broadcastedAt + (15 * 60 * 1000)); }); }); + + describe('a write that fails is not taken for one that landed', () => { + let failingCollection; + + beforeEach(() => { + failingCollection = { + findOne: sinon.stub().resolves({}), + updateOne: sinon.stub().rejects(new Error('connection timed out')), + bulkWrite: sinon.stub().rejects(new Error('connection timed out')), + }; + dbHelperStub.databaseConnection.returns({ db: () => ({ collection: () => failingCollection }) }); + }); + + const now = () => Date.now(); + // Every event a node records once and serves to the peers syncing from it. + const events = () => [ + ['apprunning', { signedBroadcast: { data: { ip: '1.2.3.4:16127', broadcastedAt: now(), version: 2, apps: [{ name: 'a' }] } } }], + ['sigterm', { message: { ip: '1.2.3.4:16127', broadcastedAt: now() } }], + ['appremoved', { message: { ip: '1.2.3.4:16127', appName: 'a', broadcastedAt: now() } }], + ['evicted', { ip: '1.2.3.4:16127', evictedAt: now() }], + ['ipchanged', { message: { oldIP: '1.2.3.4:16127', newIP: '5.6.7.8:16127', broadcastedAt: now() } }], + ]; + + it('rejects each state event it could not record', async () => { + // eslint-disable-next-line no-restricted-syntax + for (const [type, payload] of events()) { + // eslint-disable-next-line no-await-in-loop + const outcome = await messageStore.storeAppStateEvent(type, payload).catch((err) => err); + expect(outcome, `a ${type} event was taken as recorded`).to.be.an('error'); + } + }); + + it('reports a batch of sync events it could not write', async () => { + const broadcast = { + version: 1, timestamp: now(), pubKey: 'k', signature: 's', + data: { ip: '1.2.3.4:16127', name: 'a', broadcastedAt: now() }, + }; + + const installing = await messageStore.storeBatchAppInstallingMessages([broadcast]); + const errors = await messageStore.storeBatchAppInstallingErrorMessages([broadcast]); + + expect(installing.writeFailed).to.equal(true); + expect(errors.writeFailed).to.equal(true); + }); + + it('rejects a signed claim it could not record', async () => { + dbHelperStub.updateOneInDatabase.rejects(new Error('connection timed out')); + const broadcast = { + version: 1, timestamp: now(), pubKey: 'k', signature: 's', + data: { + ip: '1.2.3.4:16127', name: 'a', hash: 'h', broadcastedAt: now(), + }, + }; + + const claim = await messageStore.storeSignedAppInstallingBroadcast(broadcast).catch((err) => err); + const error = await messageStore.storeSignedAppInstallingErrorBroadcast(broadcast).catch((err) => err); + + expect(claim).to.be.an('error'); + expect(error).to.be.an('error'); + }); + }); }); From 928cad34f32c0293fbdf05c2a41d6ad21dc5db7f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:21:48 +0100 Subject: [PATCH 054/369] fix(db): an address adoption waits for a readable record of the apps this node ran installedAppsListedAt rejects when the installed apps cannot be read, and ranAppsAt answers null. Neither adoption path announces or records a move on that answer: handleOwnAddress acts on the address again after 60 s, and the collision check that calls leaveConfiguredAddress asks again on its next pass. On a machine started from a copy of another's configuration, a move announced on a guess would move that machine's location rows. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appQuery/appQueryService.js | 5 ++- ZelBack/src/services/fluxNetworkHelper.js | 38 ++++++++++++++++--- tests/unit/appQueryService.test.js | 10 +++-- tests/unit/ownAddress.test.js | 36 ++++++++++++++++++ 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index da109e3918..7fa95b26d8 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -140,13 +140,14 @@ async function decryptEnterpriseApps(apps, options = {}) { /** * The names of the apps installed on this node that the network lists at - * `socketAddress`: what a node that held that address left there. + * `socketAddress`: what a node that held that address left there. Rejects + * when either cannot be read. * @param {string} socketAddress * @returns {Promise} */ async function installedAppsListedAt(socketAddress) { const installed = await installedApps(); - if (installed.status !== 'success') return []; + if (installed.status !== 'success') throw new Error(`installed apps unreadable: ${installed.data?.message ?? installed.data}`); const names = new Set(installed.data.map((app) => app.name)); return (await registryManager.appsListedAt(socketAddress)).filter((name) => names.has(name)); } diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 19635393c1..753c1847d1 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -69,13 +69,18 @@ function setInstalledAppsListedAt(callback) { * Whether this node ran apps at `socketAddress`: an app installed here is one * the network lists there. A configuration names an address, and a machine * started on a copy of another's names that machine's; only apps left there - * show this node held it. + * show this node held it. `null` when that cannot be read. * @param {string} socketAddress - * @returns {Promise} + * @returns {Promise} */ async function ranAppsAt(socketAddress) { if (!installedAppsListedAt) return false; - return (await installedAppsListedAt(socketAddress)).length > 0; + try { + return (await installedAppsListedAt(socketAddress)).length > 0; + } catch (error) { + log.warn(`own address - whether this node ran apps at ${socketAddress} cannot be read: ${error.message}`); + return null; + } } let dosState = 0; // we can start at bigger number later @@ -2050,7 +2055,9 @@ async function fluxGoneFrom(socketAddress) { /** * Adopts `current` in place of `previous`, the configured address, which this * node has no record of leaving. Told to the network as a move only when this - * node ran apps there. + * node ran apps there. While that cannot be read nothing is adopted: the + * collision check that calls this finds the same listing on its next pass and + * asks again. * @param {string} previous * @param {string} current * @returns {Promise} @@ -2058,10 +2065,17 @@ async function fluxGoneFrom(socketAddress) { function leaveConfiguredAddress(previous, current) { return serialiseOwnAddressWork(async () => { const from = normalizeSocketAddress(previous); - await adjustExternalIp(from, current, { announceMove: await ranAppsAt(from) }); + const ran = await ranAppsAt(from); + if (ran === null) return; + await adjustExternalIp(from, current, { announceMove: ran }); }); } +// How long an adoption deferred on an unreadable database waits before this +// node's address is acted on again; the retry pending, while one is. +const OWN_ADDRESS_RETRY_MS = 60 * 1000; +let ownAddressRetry = null; + /** * Acts on the address benchmark reports as this node's own. One that differs * from the address the node last adopted is a move, whichever way it came and @@ -2071,6 +2085,8 @@ function leaveConfiguredAddress(previous, current) { * the Flux node there is gone (fluxGoneFrom) it adopts the reported address and * the collision check judges it; once it is, it adopts the reported address in * place of the configured one, told as a move only when it ran apps there. + * While whether it ran apps there cannot be read it adopts nothing, and acts on + * the address it knows again after OWN_ADDRESS_RETRY_MS. * @param {string} current The reported socket address. * @returns {Promise} */ @@ -2087,7 +2103,15 @@ function handleOwnAddress(current) { await writeOwnAddressRecord(reported); return; } - await adjustExternalIp(previous, reported, { announceMove: Boolean(record) || await ranAppsAt(previous) }); + const announceMove = record ? true : await ranAppsAt(previous); + if (announceMove === null) { + ownAddressRetry ??= setTimeout(() => { + ownAddressRetry = null; + handleOwnAddress(getKnownLocalSocketAddress() || reported); + }, OWN_ADDRESS_RETRY_MS); + return; + } + await adjustExternalIp(previous, reported, { announceMove }); } else if (!record) { await writeOwnAddressRecord(reported); } else { @@ -2115,6 +2139,8 @@ function startOwnAddressWatch() { return () => { if (ownAddressListener) offLocalSocketAddressChange(ownAddressListener); ownAddressListener = null; + clearTimeout(ownAddressRetry); + ownAddressRetry = null; }; } diff --git a/tests/unit/appQueryService.test.js b/tests/unit/appQueryService.test.js index d3a4b1f83f..67ccd60f01 100644 --- a/tests/unit/appQueryService.test.js +++ b/tests/unit/appQueryService.test.js @@ -287,13 +287,17 @@ describe('appQueryService tests', () => { expect(await appQueryService.installedAppsListedAt('198.51.100.1:16127')).to.deep.equal(['kept']); }); - it('names none when the installed apps cannot be read', async () => { + it('rejects when the installed apps cannot be read', async () => { dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('appsDatabase') }); dbHelperStub.findInDatabase.rejects(new Error('db down')); - messageHelperStub.createErrorMessage.returns({ status: 'error' }); + messageHelperStub.createErrorMessage.returns({ status: 'error', data: { message: 'db down' } }); registryManagerStub.appsListedAt.resolves(['kept']); - expect(await appQueryService.installedAppsListedAt('198.51.100.1:16127')).to.deep.equal([]); + const error = await appQueryService.installedAppsListedAt('198.51.100.1:16127').catch((caught) => caught); + + expect(error).to.be.an('error'); + expect(error.message).to.include('db down'); + sinon.assert.notCalled(registryManagerStub.appsListedAt); }); }); diff --git a/tests/unit/ownAddress.test.js b/tests/unit/ownAddress.test.js index 52d594f7ca..1fffd4d9e0 100644 --- a/tests/unit/ownAddress.test.js +++ b/tests/unit/ownAddress.test.js @@ -153,6 +153,29 @@ describe('fluxNetworkHelper own address tests', () => { expect(restarted.firstCall.args[0].map((app) => app.name)).to.deep.equal(['expiredrows']); }); + it('adopts nothing while whether it ran apps there cannot be read, and acts on the address again later', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); + configure('198.51.100.90'); + listedHere.rejects(new Error('db down')); + + await fluxNetworkHelper.handleOwnAddress('198.51.100.91:16127'); + + sinon.assert.notCalled(moved); + sinon.assert.notCalled(fs.writeFile); + expect(await record()).to.equal(null); + + listedHere.resolves(['ranthere']); + fluxNetworkHelper.setLocalSocketAddress('198.51.100.91:16127'); + const told = new Promise((resolve) => { moved.callsFake(async () => resolve()); }); + await clock.tickAsync(60 * 1000); + await told; + // queued behind the retry, so it returns once the retry has finished + await fluxNetworkHelper.handleOwnAddress('198.51.100.91:16127'); + + sinon.assert.calledOnceWithExactly(moved, '198.51.100.90:16127', '198.51.100.91:16127', sinon.match.number); + expect(await record()).to.deep.equal({ socketAddress: '198.51.100.91:16127', movedFrom: '198.51.100.90:16127' }); + }); + it('adopts the new address without a move when nothing reports what it ran', async () => { configure('198.51.100.21'); fluxNetworkHelper.setInstalledAppsListedAt(null); @@ -578,6 +601,19 @@ describe('fluxNetworkHelper own address tests', () => { expect(await record()).to.deep.equal({ socketAddress: '198.51.100.9:16157', movedFrom: '198.51.100.9:16127' }); }); + it('adopts nothing from a listing at its own configured IP that nothing answers, while whether it ran apps there cannot be read', async () => { + configure('198.51.100.92'); + collateralListedAt('198.51.100.92:16127', '198.51.100.93:16127'); + listedHere.rejects(new Error('db down')); + + await fluxNetworkHelper.checkDeterministicNodesCollisions(); + + sinon.assert.notCalled(moved); + sinon.assert.notCalled(fs.writeFile); + sinon.assert.notCalled(daemonServiceFluxnodeRpcs.createConfirmationTransaction); + expect(await record()).to.equal(null); + }); + it('adopts the new address without a move from a listing at its own configured IP that nothing answers, where it ran no apps', async () => { configure('198.51.100.25'); collateralListedAt('198.51.100.25:16127', '198.51.100.26:16127'); From ec9a46d4f1667c4f9d5d15b85980c8abfcc61fd3 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:22:45 +0100 Subject: [PATCH 055/369] fix(db): a settle marker that cannot be read keeps the time it holds getSettleMarker rejects when the marker cannot be read and answers null only when there is none. noteVerdictConfirmed writes nothing on a failed read, so the observed window it holds is not overwritten with zero, and evacuation stays off for that tick. Co-Authored-By: Claude Opus 5.5 --- .../src/services/residentialNodeDosService.js | 28 ++++++++++--------- tests/unit/residentialNodeDosService.test.js | 17 +++++++++++ 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/ZelBack/src/services/residentialNodeDosService.js b/ZelBack/src/services/residentialNodeDosService.js index a8080752b3..c2d25de8c8 100644 --- a/ZelBack/src/services/residentialNodeDosService.js +++ b/ZelBack/src/services/residentialNodeDosService.js @@ -258,7 +258,8 @@ async function isResidential() { } /** - * The settling marker as stored, or null when it cannot be read. + * The settling marker as stored, or null when there is none. Rejects when it + * cannot be read. * * Persisted rather than held in memory: a counter of consecutive evaluations in * memory is reset by restarting FluxOS, which would make restarting on a timer a @@ -266,16 +267,11 @@ async function isResidential() { * @returns {Promise} The marker document, or null. */ async function getSettleMarker() { - try { - const db = dbHelper.databaseConnection(); - if (!db) return null; - const database = db.db(config.database.local.database); - const marker = await dbHelper.findOneInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }); - return marker || null; - } catch (error) { - log.warn(`residentialNodeDos - could not read settle marker: ${error.message}`); - return null; - } + const db = dbHelper.databaseConnection(); + if (!db) throw new Error('no database connection'); + const database = db.db(config.database.local.database); + const marker = await dbHelper.findOneInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }); + return marker || null; } /** @@ -318,10 +314,16 @@ function creditForGap(gapMs) { * the router the way to postpone enforcement. * @param {number} now Epoch ms. * @returns {Promise} Observed milliseconds, or null when the marker - * cannot be written. + * cannot be read or written. A marker that cannot be read is not written. */ async function noteVerdictConfirmed(now) { - const marker = await getSettleMarker(); + let marker; + try { + marker = await getSettleMarker(); + } catch (error) { + log.warn(`residentialNodeDos - could not read settle marker: ${error.message}`); + return null; + } // A marker written before this node kept an observed total has no record of // what was watched, only of when the clock started - and that is exactly the // measure being replaced. It starts accumulating from here rather than being diff --git a/tests/unit/residentialNodeDosService.test.js b/tests/unit/residentialNodeDosService.test.js index 538cfc634d..8a3060d779 100644 --- a/tests/unit/residentialNodeDosService.test.js +++ b/tests/unit/residentialNodeDosService.test.js @@ -505,6 +505,23 @@ describe('residentialNodeDosService tests', () => { expect(markerStore.get('residentialDos').observedMs).to.equal(service.CHECK_INTERVAL_MS); }); + it('keeps the observed time and evacuates nothing when the marker cannot be read', async () => { + const now = Date.now(); + markerStore.set('residentialDos', { + _id: 'residentialDos', + residentialSince: now - (5 * 24 * HOUR), + lastConfirmedAt: now - service.CHECK_INTERVAL_MS, + observedMs: service.SETTLE_MS - HOUR, + }); + dbHelperStub.findOneInDatabase.rejects(new Error('marker read failed')); + + await service.enforceResidentialPolicy(deps); + + sinon.assert.notCalled(dbHelperStub.findOneAndUpdateInDatabase); + expect(markerStore.get('residentialDos').observedMs).to.equal(service.SETTLE_MS - HOUR); + expect(service.isEvacuating()).to.equal(false); + }); + it('starts a marker written before observed time was kept at zero, not credited', async () => { // An in-place upgrade. The old marker records when the clock started, // which is precisely the measure being replaced, so it earns nothing - From 388919fce43e2117917b638a83bb9147bacaa697 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:24:03 +0100 Subject: [PATCH 056/369] fix(db): the hash sync version marker is written only after the upgrade reset ran The once-per-upgrade hash retry reset is a step of every hash sync and hash retry pass until it succeeds, and the version marker is written only once it has. A reset that fails leaves the marker as it was, so the next pass, and the next boot, run it again. Co-Authored-By: Claude Opus 5.5 --- .../appMessaging/appSyncOrchestrator.js | 13 ++++++++++-- tests/unit/appSyncOrchestrator.test.js | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index 7d0088a098..b5544d92f2 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -164,6 +164,10 @@ class AppSyncOrchestrator { #nextHashRetryHeight = 0; #lastBlockHeight = 0; #fluxVersion = null; + + // Whether this version's once-per-upgrade hash retry reset has run, or was + // already recorded as run. The version marker is written only once it has. + #upgradeResetDone = false; #heartbeatInterval = null; #bootContext = null; #canSendMessages = false; @@ -887,6 +891,10 @@ class AppSyncOrchestrator { this.#syncInProgress = true; try { + if (!this.#upgradeResetDone) { + await this.#checkVersionUpgrade(); + await this.#writeVersionMarker(); + } const result = await appHashSyncService.syncMissingHashes({ currentHeight: this.#lastBlockHeight }); this.#nextHashRetryHeight = result.nextRetryHeight ?? (this.#lastBlockHeight + FALLBACK_RECHECK_BLOCKS); // A retry stores the messages it resolves without promoting them, so nothing else applies @@ -914,7 +922,6 @@ class AppSyncOrchestrator { return; } log.info('AppSyncOrchestrator - Sync started'); - await this.#checkVersionUpgrade(); log.info('AppSyncOrchestrator - Starting initial hash sync'); await this.#runHashSync(); this.#checkReadiness(); @@ -930,13 +937,14 @@ class AppSyncOrchestrator { const resetCount = await appHashSyncService.resetHashSyncForUpgrade(this.#lastBlockHeight); log.info(`AppSyncOrchestrator - Version upgrade to ${this.#fluxVersion}, reset ${resetCount} hash sync entries`); } + this.#upgradeResetDone = true; } catch (error) { log.error(`AppSyncOrchestrator - Version upgrade check failed: ${error.message}`); } } async #writeVersionMarker() { - if (!this.#fluxVersion) return; + if (!this.#fluxVersion || !this.#upgradeResetDone) return; try { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -956,6 +964,7 @@ class AppSyncOrchestrator { this.#syncInProgress = true; try { this.#hashSyncAttempts += 1; + if (!this.#upgradeResetDone) await this.#checkVersionUpgrade(); const result = await appHashSyncService.syncMissingHashes({ currentHeight: this.#lastBlockHeight }); if (result.missing > 0) { log.warn(`AppSyncOrchestrator - Hash sync has ${result.missing} unresolvable hashes, proceeding`); diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index f1de286633..3c935a8224 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -2274,6 +2274,27 @@ describe('AppSyncOrchestrator', () => { expect(versionCall.args[3]).to.deep.equal({ $set: { version: '8.12.0' } }); }); + it('writes no version marker while the reset fails, and runs the reset again on the next hash retry', async () => { + dbHelperStub.findOneInDatabase.resolves(null); + resetHashSyncForUpgradeStub.onFirstCall().rejects(new Error('db down')); + const versionWrites = () => findOneAndUpdateStub.getCalls().filter((c) => c.args[2]?._id === 'hashSyncVersion'); + + const orchestrator = makeOrchestrator({ fluxVersion: '8.12.0' }); + orchestrator.start(defaultBootContext); + + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + + expect(syncMissingHashesStub.calledOnce).to.be.true; + expect(versionWrites()).to.have.length(0); + + blockEmitter.emit('blocksProcessed', 2555100); + await clock.tickAsync(0); + + expect(resetHashSyncForUpgradeStub.calledTwice).to.be.true; + expect(versionWrites()).to.have.length(1); + }); + it('should skip version check when fluxVersion not provided', async () => { const orchestrator = makeOrchestrator(); orchestrator.start(defaultBootContext); From 80b00b58953e9a894a4467a1af1075459f1ef892 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:25:25 +0100 Subject: [PATCH 057/369] fix(db): a settle marker that could not be cleared or a departure time that could not be written is retried A failed clear of the settle marker is remembered, and while it stands the enforcing tick clears it again before counting the window: a verdict that flips back never inherits the window of the one before. A departure time that could not be written is written again on each enforcing tick until it is, so a restart does not reopen the departure gate early. Co-Authored-By: Claude Opus 5.5 --- .../src/services/residentialNodeDosService.js | 31 +++++++++++++--- tests/unit/residentialNodeDosService.test.js | 36 +++++++++++++++++++ 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/residentialNodeDosService.js b/ZelBack/src/services/residentialNodeDosService.js index c2d25de8c8..36504f8577 100644 --- a/ZelBack/src/services/residentialNodeDosService.js +++ b/ZelBack/src/services/residentialNodeDosService.js @@ -161,6 +161,13 @@ let lastDecision = null; // named rather than encoded in a magic origin. let lastEvacuationAt = null; +// A clear of the settle marker that failed: the marker still holds the window a +// verdict that has since flipped accumulated, and nothing may count it. +let settleMarkerUncleared = false; +// A departure whose time could not be written to the marker, written again on +// each tick until it is: the next process reopens the gate early without it. +let unpersistedEvacuationAt = null; + // Whether this node yet knows what it is running. Starts false and is only // raised by the orchestrator's own signal - the same one appSpawner waits on. // globalState.spawnerPaused is NOT this signal: it initialises to false, so @@ -379,16 +386,23 @@ async function noteVerdictConfirmed(now) { /** * Clear the settling window. Only a verdict flip does this - the node is no - * longer residential, or is now ArcaneOS. + * longer residential, or is now ArcaneOS. A clear that fails is remembered, and + * the window is not counted again until a clear succeeds. + * @returns {Promise} Whether the marker is cleared. */ async function clearSettleMarker() { try { const db = dbHelper.databaseConnection(); - if (!db) return; + if (!db) throw new Error('no database connection'); const database = db.db(config.database.local.database); await dbHelper.findOneAndDeleteInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }, {}); + settleMarkerUncleared = false; + unpersistedEvacuationAt = null; + return true; } catch (error) { + settleMarkerUncleared = true; log.warn(`residentialNodeDos - could not clear settle marker: ${error.message}`); + return false; } } @@ -560,7 +574,7 @@ function noteEvacuated(appName, now = monotonicMs()) { async function persistLastEvacuationAt(now) { try { const db = dbHelper.databaseConnection(); - if (!db) return; + if (!db) throw new Error('no database connection'); const database = db.db(config.database.local.database); await dbHelper.findOneAndUpdateInDatabase( database, @@ -569,8 +583,10 @@ async function persistLastEvacuationAt(now) { { $set: { lastEvacuationAt: now } }, { upsert: true }, ); + if (unpersistedEvacuationAt === now) unpersistedEvacuationAt = null; } catch (error) { - log.warn(`residentialNodeDos - could not record the departure time: ${error.message}`); + unpersistedEvacuationAt = Math.max(unpersistedEvacuationAt ?? 0, now); + log.warn(`residentialNodeDos - could not record the departure time, written again on the next tick: ${error.message}`); } } @@ -725,6 +741,13 @@ async function runResidentialPolicy(deps) { return true; } + if (settleMarkerUncleared && !await clearSettleMarker()) { + evacuating = false; + log.info('residentialNodeDos - the settle marker of an earlier verdict is not yet cleared, evacuation stays off'); + return false; + } + if (unpersistedEvacuationAt !== null) await persistLastEvacuationAt(unpersistedEvacuationAt); + const now = Date.now(); const observedMs = await noteVerdictConfirmed(now); if (observedMs !== null) observedWindowMs = observedMs; diff --git a/tests/unit/residentialNodeDosService.test.js b/tests/unit/residentialNodeDosService.test.js index 8a3060d779..2c17575c66 100644 --- a/tests/unit/residentialNodeDosService.test.js +++ b/tests/unit/residentialNodeDosService.test.js @@ -431,6 +431,28 @@ describe('residentialNodeDosService tests', () => { expect(service.isEvacuating()).to.equal(false); }); + it('counts nothing of a window a verdict flip could not clear, until it is cleared', async () => { + installedApps = [{ name: 'someapp' }]; + markerStore.set('residentialDos', windowServed()); + const residential = geolocationServiceStub.getNetworkClassification(); + dbHelperStub.findOneAndDeleteInDatabase.rejects(new Error('marker delete failed')); + geolocationServiceStub.getNetworkClassification.returns({ classification: CLASSIFICATION.DATACENTER }); + await service.enforceResidentialPolicy(deps); + expect(markerStore.has('residentialDos')).to.equal(true); + + geolocationServiceStub.getNetworkClassification.returns(residential); + await service.enforceResidentialPolicy(deps); + + expect(service.isEvacuating(), 'the window of the earlier verdict is not counted').to.equal(false); + expect(markerStore.get('residentialDos').observedMs).to.equal(service.SETTLE_MS + 1000); + + dbHelperStub.findOneAndDeleteInDatabase.callsFake(async (db, coll, query) => { markerStore.delete(query._id); }); + await service.enforceResidentialPolicy(deps); + + expect(service.isEvacuating()).to.equal(false); + expect(markerStore.get('residentialDos').observedMs, 'a fresh window').to.equal(0); + }); + it('stops evacuating when the marker cannot be read', async () => { installedApps = [{ name: 'someapp' }]; markerStore.set('residentialDos', windowServed()); @@ -585,6 +607,20 @@ describe('residentialNodeDosService tests', () => { expect(markerStore.get('residentialDos').lastEvacuationAt).to.be.a('number'); }); + it('writes a departure time it could not record on the next tick', async () => { + installedApps = [{ name: 'someapp' }]; + markerStore.set('residentialDos', windowServed()); + dbHelperStub.findOneAndUpdateInDatabase.onFirstCall().rejects(new Error('marker write failed')); + + service.noteEvacuated('someapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(markerStore.get('residentialDos').lastEvacuationAt).to.equal(undefined); + + await service.enforceResidentialPolicy(deps); + + expect(markerStore.get('residentialDos').lastEvacuationAt).to.be.a('number'); + }); + it('gives an empty location list the longest wait, not the shortest', async () => { // `index < 0 ? ordered.length : index` made an EMPTY list position 0 - // the front of the queue - inverting the rule the sibling test asserts. An From 210ecb9c6a1ef370e5480c457eac1550737b5ce3 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:27:23 +0100 Subject: [PATCH 058/369] fix(db): a geolocation record that cannot be read is not taken for none getGeolocationFromDb and getNodeGeolocation reject when the stored record is needed and cannot be read. The refresh pass then stops and runs again after its failure delay, so it never writes over the stored address history with a record built from nothing. /flux/info answers geolocation null while this process knows none; /flux/geolocation answers an error. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxService.js | 6 +- ZelBack/src/services/geolocationService.js | 68 +++++++++------------- tests/unit/geolocationService.test.js | 27 ++++++--- 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/ZelBack/src/services/fluxService.js b/ZelBack/src/services/fluxService.js index 390bb0cdaa..94e843c3af 100644 --- a/ZelBack/src/services/fluxService.js +++ b/ZelBack/src/services/fluxService.js @@ -1532,7 +1532,11 @@ async function getFluxInfo(req, res) { benchmark: {}, flux: {}, apps: {}, - geolocation: await geolocationService.getNodeGeolocation(), + // null while this process knows no geolocation. + geolocation: await geolocationService.getNodeGeolocation().catch((error) => { + log.warn(`flux info - geolocation unreadable: ${error.message}`); + return null; + }), }; const versionRes = await getFluxVersion(); if (versionRes.status === 'error') { diff --git a/ZelBack/src/services/geolocationService.js b/ZelBack/src/services/geolocationService.js index 723dd7b9c5..c195fca9cb 100644 --- a/ZelBack/src/services/geolocationService.js +++ b/ZelBack/src/services/geolocationService.js @@ -80,52 +80,37 @@ async function storeGeolocationToDb(geolocation, isStaticIp, isDataCenter, ipCha } /** - * Retrieves geolocation data from the database + * Retrieves geolocation data from the database. Rejects when it cannot be read. * @returns {Promise<{geolocation: object|null, staticIp: boolean, dataCenter: boolean, lastIpChangeDate: number|null}>} */ async function getGeolocationFromDb() { - try { - const dbClient = dbHelper.databaseConnection(); - if (!dbClient) { - return { geolocation: null, staticIp: false, dataCenter: false, lastIpChangeDate: null }; - } - const database = dbClient.db(config.database.local.database); - const query = { _id: 'nodeGeolocation' }; - const result = await dbHelper.findOneInDatabase(database, geolocationCollection, query); - if (result && result.geolocation) { - return { - geolocation: result.geolocation, - staticIp: result.staticIp || false, - dataCenter: result.dataCenter || false, - lastIpChangeDate: result.lastIpChangeDate || null, - ipFirstSeenAt: result.ipFirstSeenAt || null, - // `?? null`: a document written before these fields existed carries no - // observation, and saying so lets the next pass start the window. - staticIpState: result.staticIpState ?? null, - networkEvidence: result.networkEvidence ?? null, - }; - } - return { - geolocation: null, - staticIp: false, - dataCenter: false, - lastIpChangeDate: null, - ipFirstSeenAt: null, - staticIpState: null, - networkEvidence: null, - }; - } catch (error) { - log.error(`Failed to retrieve geolocation from database: ${error.message}`); + const dbClient = dbHelper.databaseConnection(); + if (!dbClient) throw new Error('no database connection'); + const database = dbClient.db(config.database.local.database); + const query = { _id: 'nodeGeolocation' }; + const result = await dbHelper.findOneInDatabase(database, geolocationCollection, query); + if (result && result.geolocation) { return { - geolocation: null, - staticIp: false, - dataCenter: false, - lastIpChangeDate: null, - ipFirstSeenAt: null, - staticIpState: null, - networkEvidence: null, + geolocation: result.geolocation, + staticIp: result.staticIp || false, + dataCenter: result.dataCenter || false, + lastIpChangeDate: result.lastIpChangeDate || null, + ipFirstSeenAt: result.ipFirstSeenAt || null, + // `?? null`: a document written before these fields existed carries no + // observation, and saying so lets the next pass start the window. + staticIpState: result.staticIpState ?? null, + networkEvidence: result.networkEvidence ?? null, }; } + return { + geolocation: null, + staticIp: false, + dataCenter: false, + lastIpChangeDate: null, + ipFirstSeenAt: null, + staticIpState: null, + networkEvidence: null, + }; } /** @@ -559,7 +544,8 @@ async function setNodeGeolocation() { * the node's own record of where it is, and the next refresh persists whatever * it holds. The record is flat scalars, so one level copies all of it - a * nested field added later needs this to deepen with it. - * @returns {Promise} The geolocation object or null + * @returns {Promise} The geolocation object, or null when none is + * known. Rejects when the stored record is needed and cannot be read. */ async function getNodeGeolocation() { if (storedGeolocation) { diff --git a/tests/unit/geolocationService.test.js b/tests/unit/geolocationService.test.js index 49fbfd3e6f..2229b3d49f 100644 --- a/tests/unit/geolocationService.test.js +++ b/tests/unit/geolocationService.test.js @@ -229,21 +229,20 @@ describe('geolocationService tests', () => { expect(geolocationService.getLastIpChangeDate()).to.equal(mockDbResult.lastIpChangeDate); }); - it('should return defaults when db connection is not available', async () => { + it('rejects when db connection is not available', async () => { dbHelperStub.databaseConnection.returns(null); - const result = await geolocationService.getNodeGeolocation(); + const error = await geolocationService.getNodeGeolocation().catch((caught) => caught); - expect(result).to.equal(null); + expect(error).to.be.an('error'); }); - it('should handle db error gracefully', async () => { + it('rejects when the stored record cannot be read', async () => { dbHelperStub.findOneInDatabase.rejects(new Error('DB error')); - const result = await geolocationService.getNodeGeolocation(); + const error = await geolocationService.getNodeGeolocation().catch((caught) => caught); - expect(result).to.equal(null); - sinon.assert.calledOnce(logStub.error); + expect(error.message).to.equal('DB error'); }); }); @@ -510,6 +509,18 @@ describe('geolocationService tests', () => { expect(geolocationService.isStaticIP()).to.equal(false); }); + it('writes nothing over a stored record it cannot read, and runs the pass again later', async () => { + fluxNetworkHelperStub.hasPublicIpOnInterface.resolves(true); + serviceHelperStub.axiosGet.resolves(ipApiResponse()); + dbHelperStub.findOneInDatabase.rejects(new Error('DB error')); + geolocationService = reload(); + + await geolocationService.setNodeGeolocation(); + + sinon.assert.notCalled(dbHelperStub.updateOneInDatabase); + expect(await geolocationService.getNodeGeolocation().catch(() => 'unreadable')).to.equal('unreadable'); + }); + it('seeds the window from the last recorded change, so an upgrade costs nothing', async () => { // ipFirstSeenAt is new, so an in-place upgrade has none. lastIpChangeDate // is persisted, restored, and says the address has been held since that @@ -1296,7 +1307,7 @@ describe('geolocationService tests', () => { await geolocationService.setNodeGeolocation(); - sinon.assert.calledWith(logStub.warn, 'Database connection not available for storing geolocation'); + sinon.assert.notCalled(dbHelperStub.updateOneInDatabase); }); it('should handle db storage error gracefully', async () => { From 8ee783f4ae139b6555182e3cd22c5317b3f305bf Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:28:52 +0100 Subject: [PATCH 059/369] fix(db): a spawn attempt the database stopped is tried again on the next pass A database error in trySpawningGlobalApplication clears the app's hash from the spawn cache, so the next pass considers the app again. Any other error keeps the hash cached as before. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appSpawner.js | 6 +++ tests/unit/appSpawner.test.js | 37 ++++++++++++++++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/appLifecycle/appSpawner.js b/ZelBack/src/services/appLifecycle/appSpawner.js index d3cd50f53b..75176f1441 100644 --- a/ZelBack/src/services/appLifecycle/appSpawner.js +++ b/ZelBack/src/services/appLifecycle/appSpawner.js @@ -1178,6 +1178,12 @@ async function trySpawningGlobalApplication() { return isEnterprise ? 0 : delayTime; } catch (error) { log.error(error); + // A database that did not answer says nothing about the app: the next pass + // tries it again. + if (appHash && dbHelper.isDatabaseError(error)) { + globalState.trySpawningGlobalAppCache.delete(appHash); + return shortDelayTime || 5 * 60 * 1000; + } if (appHash && !globalState.spawnErrorsLongerAppCache.has(appHash) && !globalState.trySpawningGlobalAppCache.has(appHash)) { log.info(`trySpawningGlobalApplication - Adding app hash ${appHash} to trySpawningGlobalAppCache due to pre-install error`); globalState.trySpawningGlobalAppCache.set(appHash, '', { ttl: FluxCacheManager.oneHour * 6 }); diff --git a/tests/unit/appSpawner.test.js b/tests/unit/appSpawner.test.js index 0365fe4b5c..a63a813565 100644 --- a/tests/unit/appSpawner.test.js +++ b/tests/unit/appSpawner.test.js @@ -2,6 +2,8 @@ const { expect } = require('chai'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); +const { MongoNetworkTimeoutError } = require('mongodb'); +const { isDatabaseError } = require('../../ZelBack/src/services/dbHelper'); const realPlacementFeasibility = require('../../ZelBack/src/services/appPlacement/placementFeasibility'); const realImageManager = require('../../ZelBack/src/services/appSecurity/imageManager'); const { resetGlobalState } = require('./fixtures/globalState'); @@ -125,7 +127,8 @@ describe('appSpawner tests', () => { '../dbHelper': { databaseConnection: sinon.stub().returns({ db: sinon.stub().returns({}) }), aggregateInDatabase: aggregateStub, - findInDatabase: sinon.stub().resolves([]), + findInDatabase: opts.localAppsRead ?? sinon.stub().resolves([]), + isDatabaseError, }, '../serviceHelper': { delay: delayStub, @@ -1052,6 +1055,38 @@ describe('appSpawner tests', () => { expect(globalStateStub.spawnErrorsLongerAppCache.has('abc123')).to.be.false; }); + it('tries the app again on the next pass when the installed apps cannot be read', async () => { + const installStub = sinon.stub().resolves(InstallOutcome.INSTALLED); + buildModule({ + aggregateResult: [spawnableApp], + appSpec: fullSpec, + errorCount: 0, + installStub, + localAppsRead: sinon.stub().rejects(new MongoNetworkTimeoutError('timed out')), + }); + + await appSpawner.trySpawningGlobalApplication().catch(() => {}); + + sinon.assert.notCalled(installStub); + expect(globalStateStub.trySpawningGlobalAppCache.has('abc123')).to.be.false; + }); + + it('holds the app back after an error that is not the database', async () => { + const installStub = sinon.stub().resolves(InstallOutcome.INSTALLED); + buildModule({ + aggregateResult: [spawnableApp], + appSpec: fullSpec, + errorCount: 0, + installStub, + localAppsRead: sinon.stub().rejects(new Error('bad record')), + }); + + await appSpawner.trySpawningGlobalApplication().catch(() => {}); + + sinon.assert.notCalled(installStub); + expect(globalStateStub.trySpawningGlobalAppCache.has('abc123')).to.be.true; + }); + it('should not block when network error count < 5', async () => { buildModule({ aggregateResult: [spawnableApp], appSpec: fullSpec, errorCount: 4 }); await appSpawner.trySpawningGlobalApplication().catch(() => {}); From 61b185c0a26822b39e2c3da9b8b6f0d698f3fc90 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:32:34 +0100 Subject: [PATCH 060/369] fix(db): a move whose proof cannot be read is held, not refused lastSignerAt rejects when the rows at the address a move left cannot be read, and verification answers VerifyResult.UNREADABLE for it, which does not count against the relaying peer. A live broadcast with that verdict keeps its slot and is dispatched again after 30 s while it is within its window. A sync response carrying one is not credited, so another peer is asked. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxCommunication.js | 29 ++++++++++- .../src/services/fluxCommunicationUtils.js | 50 ++++++++++--------- tests/unit/fluxCommunication.test.js | 34 +++++++++++++ tests/unit/moveSignerProof.test.js | 29 +++++++++++ 4 files changed, 118 insertions(+), 24 deletions(-) diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index f2ee98cc4d..b0c7685491 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -80,6 +80,14 @@ async function handleAppMessages(message, fromIP, port) { } } +/** + * The broadcasts of a sync response that verify. Rejects when any of them + * could not be verified because the database did not answer: the response is + * then not all recorded, and its peer has not answered. + * @param {object[]} broadcasts + * @param {string} label + * @returns {Promise} + */ async function batchVerifyBroadcasts(broadcasts, label) { if (broadcasts.length === 0) return []; @@ -87,6 +95,8 @@ async function batchVerifyBroadcasts(broadcasts, label) { const verdicts = await fluxCommunicationUtils.verifyBroadcasts(broadcasts, { verifySignatures: (items) => verifyPool.verify(items), }); + const unreadable = verdicts.filter(({ result }) => result === VerifyResult.UNREADABLE).length; + if (unreadable > 0) throw new Error(`${label} - ${unreadable} broadcasts could not be verified, the database did not answer`); const verified = broadcasts.filter((broadcast, i) => verdicts[i].result === VerifyResult.OK); const signatureFailures = verdicts.filter(({ result }) => result === VerifyResult.BAD_SIGNATURE).length; const nodeCheckFailures = broadcasts.length - verified.length - signatureFailures; @@ -801,6 +811,10 @@ async function handleNodeSigtermMessage(message, fromIP, port) { // byte budget is ever wanted it belongs in lruRateLimit, covering every type, not in a // second counter here. +// How long a broadcast this node could not verify, because its database did +// not answer, waits before it is dispatched again. +const BROADCAST_VERIFY_RETRY_MS = 30 * 1000; + async function dispatchFluxMessage(msgObj, peerSocket) { const codes = peerSocket.closeCodes; const { @@ -890,7 +904,20 @@ async function dispatchFluxMessage(msgObj, peerSocket) { const { VerifyResult } = fluxCommunicationUtils; const verifyResult = await fluxCommunicationUtils.verifyFluxBroadcast(msgObj, undefined, currentTimeStamp); - if (verifyResult === VerifyResult.OK) { + if (verifyResult === VerifyResult.UNREADABLE) { + // The slot stays claimed while the broadcast waits, so its copies are not + // verified meanwhile; it is dispatched afresh while it is within its window. + if (fluxCommunicationUtils.verifyTimestampInFluxBroadcast(msgObj, currentTimeStamp + BROADCAST_VERIFY_RETRY_MS)) { + log.warn(`Deferring a ${msgObj.data?.type} from ${peerSocket.direction} peer ${peerSocket.key}: the database did not answer`); + setTimeout(() => { + releaseSlot(); + dispatchFluxMessage(msgObj, peerSocket).catch((error) => log.error(error)); + }, BROADCAST_VERIFY_RETRY_MS); + } else { + releaseSlot(); + log.warn(`Dropping a ${msgObj.data?.type} from ${peerSocket.direction} peer ${peerSocket.key}: the database did not answer within its window`); + } + } else if (verifyResult === VerifyResult.OK) { const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(msgObj, currentTimeStamp); if (timestampOK === true) { const handler = handlerFor(msgObj.data.type); diff --git a/ZelBack/src/services/fluxCommunicationUtils.js b/ZelBack/src/services/fluxCommunicationUtils.js index b952664555..1fb623e908 100644 --- a/ZelBack/src/services/fluxCommunicationUtils.js +++ b/ZelBack/src/services/fluxCommunicationUtils.js @@ -106,6 +106,7 @@ const VerifyResult = Object.freeze({ NODE_NOT_FOUND: 'nodeNotFound', BAD_SIGNATURE: 'badSignature', PUBKEY_MISMATCH: 'pubkeyMismatch', + UNREADABLE: 'unreadable', }); /** @@ -136,28 +137,23 @@ function signerAddresses(payload, target) { * only when an apprunning came after its eviction and after any move away from * it, so the newest apprunning is the one that wrote them. * @param {string} address - * @returns {Promise} Null when no rows stand there, no apprunning - * is stored for it, or either cannot be read. + * @returns {Promise} Null when no rows stand there or no apprunning + * is stored for it. Rejects when either cannot be read. */ async function lastSignerAt(address) { - try { - const database = dbHelper.databaseConnection().db(config.database.appsglobal.database); - const row = await database.collection(config.database.appsglobal.collections.appsLocations) - .findOne({ ip: address }, { projection: { _id: 0, ip: 1 } }); - if (!row) return null; - const [event] = await database.collection(config.database.appsglobal.collections.appStateEvents) - .find( - { ip: address, type: 'apprunning', expireAt: { $gt: new Date() } }, - { projection: { _id: 0, 'envelope.pubKey': 1 } }, - ) - .sort({ broadcastedAt: -1 }) - .limit(1) - .toArray(); - return event?.envelope?.pubKey ?? null; - } catch (error) { - log.error(`lastSignerAt ${address}: ${error.message}`); - return null; - } + const database = dbHelper.databaseConnection().db(config.database.appsglobal.database); + const row = await database.collection(config.database.appsglobal.collections.appsLocations) + .findOne({ ip: address }, { projection: { _id: 0, ip: 1 } }); + if (!row) return null; + const [event] = await database.collection(config.database.appsglobal.collections.appStateEvents) + .find( + { ip: address, type: 'apprunning', expireAt: { $gt: new Date() } }, + { projection: { _id: 0, 'envelope.pubKey': 1 } }, + ) + .sort({ broadcastedAt: -1 }) + .limit(1) + .toArray(); + return event?.envelope?.pubKey ?? null; } /** @@ -206,6 +202,7 @@ async function listedSignerVerdict(addresses, pubKey) { * standing pubKey OK * standing another key PUBKEY_MISMATCH * none - NODE_NOT_FOUND + * unreadable - UNREADABLE * * With no rows at the address a move left, the move has nothing to carry, and * the chain already lists its node where it is. @@ -214,7 +211,12 @@ async function listedSignerVerdict(addresses, pubKey) { * @returns {Promise<{result: string, reason?: string}>} */ async function lastSignerVerdict(address, pubKey) { - const lastSigner = await lastSignerAt(address); + let lastSigner; + try { + lastSigner = await lastSignerAt(address); + } catch (error) { + return { result: VerifyResult.UNREADABLE, reason: `The location rows at ${address}, which a move signed by ${pubKey} left, cannot be read: ${error.message}` }; + } if (lastSigner === null) { return { result: VerifyResult.NODE_NOT_FOUND, reason: `No location rows stand at ${address}, which a move signed by ${pubKey} left` }; } @@ -312,8 +314,10 @@ function verifySignaturesInline(items) { * signature, then anything that reads the database. A broadcast whose signature * fails never costs a database read. * - * NODE_NOT_FOUND is the one refusal that does not count against the peer that - * relayed the broadcast: the list is cached, and can lag the chain. + * NODE_NOT_FOUND and UNREADABLE are the refusals that do not count against the + * peer that relayed the broadcast: the list is cached, and can lag the chain, + * and UNREADABLE is this node's database not answering, which says nothing of + * the broadcast. * @param {FluxNetworkMessage[]} broadcasts * @param {object} [options] * @param {function(object[]): (boolean[]|Promise)} [options.verifySignatures] diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index 3e8a5d558a..a97b059eaa 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -484,6 +484,40 @@ describe('fluxCommunication tests', () => { ).to.equal(false); }); + it('holds a message its database could not verify and dispatches it again, once, later', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); + const data = { + type: 'fluxipchanged', version: 1, oldIP: '1.1.1.3', newIP: '2.2.2.3', broadcastedAt: Date.now(), + }; + verify.onFirstCall().resolves(fluxCommunicationUtils.VerifyResult.UNREADABLE); + + await peerManager.messageDispatcher(envelope(data), peerSocket); + await peerManager.messageDispatcher(envelope(data), peerSocket); + + expect(cacheManager.announcementSeen.has(objectHash(data))).to.equal(true); + expect(verify.callCount, 'a copy waits with it').to.equal(1); + sinon.assert.notCalled(messageStore.storeIPChangedMessage); + + await clock.tickAsync(30 * 1000); + await clock.tickAsync(100); + + expect(verify.callCount).to.equal(2); + sinon.assert.calledOnce(messageStore.storeIPChangedMessage); + }); + + it('gives the slot back for a message its database could not verify within its window', async () => { + const data = { + type: 'fluxipchanged', version: 1, oldIP: '1.1.1.4', newIP: '2.2.2.4', broadcastedAt: Date.now(), + }; + verify.resolves(fluxCommunicationUtils.VerifyResult.UNREADABLE); + fluxCommunicationUtils.verifyTimestampInFluxBroadcast.returns(false); + + await peerManager.messageDispatcher(envelope(data), peerSocket); + + expect(cacheManager.announcementSeen.has(objectHash(data))).to.equal(false); + expect(peerSocket.badMessageTimestamps).to.deep.equal([]); + }); + // The canary: the same type, verifying, DOES hold it - so the release above is the // verification failing and not the filter having been switched off. it('keeps the slot for a message that verifies', async () => { diff --git a/tests/unit/moveSignerProof.test.js b/tests/unit/moveSignerProof.test.js index 59fb05e7e1..ddcc71bf51 100644 --- a/tests/unit/moveSignerProof.test.js +++ b/tests/unit/moveSignerProof.test.js @@ -10,6 +10,8 @@ const messageStore = require('../../ZelBack/src/services/appMessaging/messageSto const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); const verifyPool = require('../../ZelBack/src/services/utils/verifyPool'); const { peerManager } = require('../../ZelBack/src/services/utils/peerState'); +const { MongoNetworkTimeoutError } = require('mongodb'); +const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); const { requireMongo } = require('./dbTestHelper'); const { VerifyResult } = fluxCommunicationUtils; @@ -104,6 +106,15 @@ describe('the proof a move\'s signer spoke for the address it left', () => { expect(await fluxCommunicationUtils.verifyFluxBroadcast(await move())).to.equal(VerifyResult.NODE_NOT_FOUND); }); + it('is unreadable, not dropped, when the rows at the address it left cannot be read', async () => { + await ranAt(OLD, pubKey); + sinon.stub(dbHelper, 'databaseConnection').returns({ + db: () => ({ collection: () => ({ findOne: sinon.stub().rejects(new MongoNetworkTimeoutError('timed out')) }) }), + }); + + expect(await fluxCommunicationUtils.verifyFluxBroadcast(await move())).to.equal(VerifyResult.UNREADABLE); + }); + it('is dropped when the only apprunning stored for the address it left has expired', async () => { await ranAt(OLD, pubKey, { ago: 2 * HOUR }); @@ -263,6 +274,24 @@ describe('the proof a move\'s signer spoke for the address it left', () => { expect(await events.countDocuments({ ip: OLD, type: 'ipchanged' })).to.equal(0); }); + it('does not credit a response carrying a move whose proof cannot be read', async () => { + const completed = sinon.spy(); + appSyncEvents.on(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, completed); + try { + await sync([]); + sinon.assert.calledOnce(completed); + + sinon.stub(dbHelper, 'databaseConnection').returns({ + db: () => ({ collection: () => ({ findOne: sinon.stub().rejects(new MongoNetworkTimeoutError('timed out')) }) }), + }); + await sync([movedAt(Date.now() - 2 * MINUTE)]); + + sinon.assert.calledOnce(completed); + } finally { + appSyncEvents.off(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, completed); + } + }); + it('holds no rows to carry from a response sent once nothing is listed at the address the node left', async () => { await sync([apprunningAt(OLD, pubKey, Date.now() - 10 * MINUTE), movedAt(Date.now() - 2 * MINUTE)]); From 8d6141b38cd667b909c66319b05a60774b43574f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:33:53 +0100 Subject: [PATCH 061/369] fix(db): a network link sweep whose read failed runs again reconnectLinkedApps and reconcileAllAppNetworkLinks run again when the installed apps cannot be read, after 30 s, 60 s, 5 min and then every 15 min until a read succeeds, with one retry pending per sweep. Co-Authored-By: Claude Opus 5.5 --- .../services/appLifecycle/appNetworkLinker.js | 42 ++++++++++++++++--- tests/unit/appNetworkLinker.test.js | 26 +++++++++++- 2 files changed, 60 insertions(+), 8 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appNetworkLinker.js b/ZelBack/src/services/appLifecycle/appNetworkLinker.js index 8c18bf2c09..a73565f30c 100644 --- a/ZelBack/src/services/appLifecycle/appNetworkLinker.js +++ b/ZelBack/src/services/appLifecycle/appNetworkLinker.js @@ -26,6 +26,30 @@ const dockerService = require('../dockerService'); const log = require('../../lib/log'); const { localAppsInformation, APP_NAME_REGEX } = require('../utils/appConstants'); +// The delays before a sweep whose installed-apps read failed runs again, by +// attempt; the last repeats until the read succeeds. One retry is pending per +// sweep at a time. +const READ_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; +const pendingRetries = new Map(); + +/** + * Runs `sweep(attempt + 1)` once the delay for `attempt` has passed, unless a + * retry of the same sweep is pending already. + * @param {string} key Names the sweep. + * @param {function(number): Promise} sweep + * @param {number} attempt + */ +function retrySweep(key, sweep, attempt) { + if (pendingRetries.has(key)) return; + const delay = READ_RETRY_MS[Math.min(attempt, READ_RETRY_MS.length - 1)]; + const timer = setTimeout(() => { + pendingRetries.delete(key); + sweep(attempt + 1).catch((error) => log.error(`${key}: ${error.message}`)); + }, delay); + if (timer.unref) timer.unref(); + pendingRetries.set(key, timer); +} + /** * Parses the `networkWith:[...]` token out of an app description. * @@ -127,19 +151,22 @@ async function connectComponentToLinkedApps(componentContainerName, fullAppSpecs /** * After an app's private network is (re)created, reconnects every locally * installed app that is networked with it back onto that network. Best-effort — - * never throws, so a redeploy is not aborted by a reconnect hiccup. + * never throws, so a redeploy is not aborted by a reconnect hiccup. When the + * installed apps cannot be read it runs again later (READ_RETRY_MS). * * @param {string} appName - the app whose network was (re)created + * @param {number} [attempt] - how many reads failed before this one * @returns {Promise} */ -async function reconnectLinkedApps(appName) { +async function reconnectLinkedApps(appName, attempt = 0) { let installedApps; try { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); installedApps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, {}, { projection: { _id: 0 } }); } catch (error) { - log.error(`reconnectLinkedApps: failed to read installed apps for ${appName}: ${error.message}`); + log.error(`reconnectLinkedApps: failed to read installed apps for ${appName}, retrying later: ${error.message}`); + retrySweep(`reconnectLinkedApps ${appName}`, (next) => reconnectLinkedApps(appName, next), attempt); return; } @@ -174,18 +201,21 @@ async function reconnectLinkedApps(appName) { /** * Boot-time sweep: ensures every installed app that declares network links is - * attached to each linked app's network. Idempotent and best-effort. + * attached to each linked app's network. Idempotent and best-effort. When the + * installed apps cannot be read it runs again later (READ_RETRY_MS). * + * @param {number} [attempt] - how many reads failed before this one * @returns {Promise} */ -async function reconcileAllAppNetworkLinks() { +async function reconcileAllAppNetworkLinks(attempt = 0) { let installedApps; try { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); installedApps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, {}, { projection: { _id: 0 } }); } catch (error) { - log.error(`reconcileAllAppNetworkLinks: failed to read installed apps: ${error.message}`); + log.error(`reconcileAllAppNetworkLinks: failed to read installed apps, retrying later: ${error.message}`); + retrySweep('reconcileAllAppNetworkLinks', reconcileAllAppNetworkLinks, attempt); return; } diff --git a/tests/unit/appNetworkLinker.test.js b/tests/unit/appNetworkLinker.test.js index 51b82d838f..f0f87b7669 100644 --- a/tests/unit/appNetworkLinker.test.js +++ b/tests/unit/appNetworkLinker.test.js @@ -175,9 +175,17 @@ describe('appNetworkLinker tests', () => { expect(dockerServiceStub.appDockerNetworkConnect.calledWith('fluxweb_appC')).to.equal(false); }); - it('does not throw when the database read fails', async () => { + it('does not throw when the database read fails, and reconnects once a later read succeeds', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); dbHelperStub.findInDatabase.rejects(new Error('db down')); await expect(appNetworkLinker.reconnectLinkedApps('appA')).to.not.be.rejected; + await expect(appNetworkLinker.reconnectLinkedApps('appA')).to.not.be.rejected; + + dbHelperStub.findInDatabase.resolves([{ name: 'appB', description: 'networkWith:[appA]' }]); + dockerServiceStub.getAppContainerNames.withArgs('appB').resolves(['fluxweb_appB']); + await clock.tickAsync(30 * 1000); + + sinon.assert.calledOnceWithExactly(dockerServiceStub.appDockerNetworkConnect, 'fluxweb_appB', 'fluxDockerNetwork_appA'); }); }); @@ -277,9 +285,23 @@ describe('appNetworkLinker tests', () => { sinon.assert.calledWith(dockerServiceStub.appDockerNetworkConnect, 'fluxweb_appB', 'fluxDockerNetwork_appA'); }); - it('does not throw when the database read fails', async () => { + it('does not throw when the database read fails, and retries with a growing delay until a read succeeds', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); dbHelperStub.findInDatabase.rejects(new Error('db down')); await expect(appNetworkLinker.reconcileAllAppNetworkLinks()).to.not.be.rejected; + + await clock.tickAsync(30 * 1000); + expect(dbHelperStub.findInDatabase.callCount).to.equal(2); + await clock.tickAsync(30 * 1000); + expect(dbHelperStub.findInDatabase.callCount, 'the second retry waits 60 s').to.equal(2); + + dbHelperStub.findInDatabase.resolves([{ name: 'appB', description: 'networkWith:[appA]' }]); + dockerServiceStub.getAppContainerNames.withArgs('appB').resolves(['fluxweb_appB']); + await clock.tickAsync(30 * 1000); + + sinon.assert.calledOnceWithExactly(dockerServiceStub.appDockerNetworkConnect, 'fluxweb_appB', 'fluxDockerNetwork_appA'); + await clock.tickAsync(60 * 60 * 1000); + expect(dbHelperStub.findInDatabase.callCount, 'nothing runs after a read succeeds').to.equal(3); }); }); }); From 1e7ae8a2a6e3dcde0f37544b4e90dd12546c90f0 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:34:58 +0100 Subject: [PATCH 062/369] fix(db): a runtime state collection that could not be prepared fails boot's schema step appsRuntimeState.prepareCollection rejects when its read, its merge or its unique index fails, as the other schema steps of boot do, so boot prepares the collection again instead of running with twins and no unique index. Co-Authored-By: Claude Opus 5.5 --- .../appManagement/appsRuntimeState.js | 142 +++++++++--------- tests/unit/appsRuntimeState.test.js | 12 +- 2 files changed, 81 insertions(+), 73 deletions(-) diff --git a/ZelBack/src/services/appManagement/appsRuntimeState.js b/ZelBack/src/services/appManagement/appsRuntimeState.js index c197be4b83..717e693611 100644 --- a/ZelBack/src/services/appManagement/appsRuntimeState.js +++ b/ZelBack/src/services/appManagement/appsRuntimeState.js @@ -682,83 +682,81 @@ async function getVolumeImage(rawIdentifier) { * the backoff history on the other - so dedupe must merge field-wise rather * than keep one doc whole: dropping a doc could drop a real operator lock, * whose loss would auto-start a deliberately stopped app. + * + * Rejects when any step fails, so boot's schema preparation runs it again. */ async function prepareCollection() { - try { - const database = collection(); - const all = await dbHelper.findInDatabase(database, appsRuntimeState, {}, {}); - const byIdentifier = new Map(); - // eslint-disable-next-line no-restricted-syntax - for (const doc of all) { - const list = byIdentifier.get(doc.identifier) || []; - list.push(doc); - byIdentifier.set(doc.identifier, list); - } - // eslint-disable-next-line no-restricted-syntax - for (const [identifier, twins] of byIdentifier) { - if (twins.length > 1) { - // Oldest first, so every field any twin carries survives and the newest - // value of each wins. Fields scatter across twins - that is the whole - // reason this merge exists - so taking them from one document would - // drop exactly what the scatter put on the other, and a field added to - // this document later would be lost without ever being named here. - // What follows overrides the ones whose correct value is not simply - // "whichever was written last". - const oldestFirst = [...twins].sort((a, b) => (a.updatedAt || 0) - (b.updatedAt || 0)); - // The image record is one claim - that the file at this path carries - // this UUID - so it moves as a pair. Merged field-wise it could take - // the path from one twin and the stamp from another, and a record - // saying an image carries a UUID it never carried refuses that volume - // for good: the refusal returns before the line that would replace it. - const withImage = oldestFirst.filter((t) => t.volumeImagePath); - const image = withImage.length ? withImage[withImage.length - 1] : null; - const merged = { - ...Object.assign({}, ...oldestFirst), - ...(image ? { volumeImagePath: image.volumeImagePath, volumeFsUuid: image.volumeFsUuid || null } : {}), - identifier, - // a lock anywhere is a lock: never auto-start a deliberately stopped app - operatorStopped: twins.some((t) => t.operatorStopped === true), - // a force anywhere is a force: a kill must never be merged down into a - // graceful stop the operator did not ask for - operatorStopForce: twins.some((t) => t.operatorStopForce === true), - // a start waits on the election only if no locked twin withholds it: a - // stop without one is the operator's later word - operatorStartRequested: twins.some((t) => t.operatorStopped === true) - && twins.every((t) => t.operatorStopped !== true || t.operatorStartRequested === true), - // the highest request wins and the lowest actuation does, so a restart - // asked for on either doc still bounces the container once - restartGeneration: Math.max(0, ...twins.map((t) => t.restartGeneration || 0)), - actuatedRestartGeneration: Math.min(...twins.map((t) => t.actuatedRestartGeneration || 0)), - // likewise, a heal-removal anywhere means a heal may be mid-flight: keep - // the reconciler on the recreate path rather than the uninstall path - networkHealRemoval: twins.some((t) => t.networkHealRemoval === true), - networkHealHistory: [...new Set(twins.flatMap((t) => t.networkHealHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), - restartHistory: [...new Set(twins.flatMap((t) => t.restartHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), - autoRestartWindow: [...new Set(twins.flatMap((t) => t.autoRestartWindow || []))].sort((a, b) => a - b).slice(-RESTART_BURST_COUNT), - updatedAt: Math.max(...twins.map((t) => t.updatedAt || 0)), - }; - // the read projects _id away, but a spread of whatever the documents - // carry must not be the thing that reintroduces it into a $set - delete merged._id; - const newestExit = twins.filter((t) => t.lastDiedAt !== undefined).sort((a, b) => b.lastDiedAt - a.lastDiedAt)[0]; - if (newestExit) { - merged.lastExitCode = newestExit.lastExitCode; - merged.lastDiedAt = newestExit.lastDiedAt; - } - log.warn(`appsRuntimeState - merged ${twins.length} duplicate docs for ${identifier} (operatorStopped=${merged.operatorStopped}, ${merged.restartHistory.length} restart entries)`); - // The merge lands on one twin before the others go, so a failure part way - // leaves duplicates for the next start to merge, never no record at all. - const kept = oldestFirst[oldestFirst.length - 1]._id; - // eslint-disable-next-line no-await-in-loop - await dbHelper.updateOneInDatabase(database, appsRuntimeState, { _id: kept }, { $set: merged }); - // eslint-disable-next-line no-await-in-loop - await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier, _id: { $ne: kept } }); + const database = collection(); + const all = await dbHelper.findInDatabase(database, appsRuntimeState, {}, {}); + const byIdentifier = new Map(); + // eslint-disable-next-line no-restricted-syntax + for (const doc of all) { + const list = byIdentifier.get(doc.identifier) || []; + list.push(doc); + byIdentifier.set(doc.identifier, list); + } + // eslint-disable-next-line no-restricted-syntax + for (const [identifier, twins] of byIdentifier) { + if (twins.length > 1) { + // Oldest first, so every field any twin carries survives and the newest + // value of each wins. Fields scatter across twins - that is the whole + // reason this merge exists - so taking them from one document would + // drop exactly what the scatter put on the other, and a field added to + // this document later would be lost without ever being named here. + // What follows overrides the ones whose correct value is not simply + // "whichever was written last". + const oldestFirst = [...twins].sort((a, b) => (a.updatedAt || 0) - (b.updatedAt || 0)); + // The image record is one claim - that the file at this path carries + // this UUID - so it moves as a pair. Merged field-wise it could take + // the path from one twin and the stamp from another, and a record + // saying an image carries a UUID it never carried refuses that volume + // for good: the refusal returns before the line that would replace it. + const withImage = oldestFirst.filter((t) => t.volumeImagePath); + const image = withImage.length ? withImage[withImage.length - 1] : null; + const merged = { + ...Object.assign({}, ...oldestFirst), + ...(image ? { volumeImagePath: image.volumeImagePath, volumeFsUuid: image.volumeFsUuid || null } : {}), + identifier, + // a lock anywhere is a lock: never auto-start a deliberately stopped app + operatorStopped: twins.some((t) => t.operatorStopped === true), + // a force anywhere is a force: a kill must never be merged down into a + // graceful stop the operator did not ask for + operatorStopForce: twins.some((t) => t.operatorStopForce === true), + // a start waits on the election only if no locked twin withholds it: a + // stop without one is the operator's later word + operatorStartRequested: twins.some((t) => t.operatorStopped === true) + && twins.every((t) => t.operatorStopped !== true || t.operatorStartRequested === true), + // the highest request wins and the lowest actuation does, so a restart + // asked for on either doc still bounces the container once + restartGeneration: Math.max(0, ...twins.map((t) => t.restartGeneration || 0)), + actuatedRestartGeneration: Math.min(...twins.map((t) => t.actuatedRestartGeneration || 0)), + // likewise, a heal-removal anywhere means a heal may be mid-flight: keep + // the reconciler on the recreate path rather than the uninstall path + networkHealRemoval: twins.some((t) => t.networkHealRemoval === true), + networkHealHistory: [...new Set(twins.flatMap((t) => t.networkHealHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), + restartHistory: [...new Set(twins.flatMap((t) => t.restartHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), + autoRestartWindow: [...new Set(twins.flatMap((t) => t.autoRestartWindow || []))].sort((a, b) => a - b).slice(-RESTART_BURST_COUNT), + updatedAt: Math.max(...twins.map((t) => t.updatedAt || 0)), + }; + // the read projects _id away, but a spread of whatever the documents + // carry must not be the thing that reintroduces it into a $set + delete merged._id; + const newestExit = twins.filter((t) => t.lastDiedAt !== undefined).sort((a, b) => b.lastDiedAt - a.lastDiedAt)[0]; + if (newestExit) { + merged.lastExitCode = newestExit.lastExitCode; + merged.lastDiedAt = newestExit.lastDiedAt; } + log.warn(`appsRuntimeState - merged ${twins.length} duplicate docs for ${identifier} (operatorStopped=${merged.operatorStopped}, ${merged.restartHistory.length} restart entries)`); + // The merge lands on one twin before the others go, so a failure part way + // leaves duplicates for the next start to merge, never no record at all. + const kept = oldestFirst[oldestFirst.length - 1]._id; + // eslint-disable-next-line no-await-in-loop + await dbHelper.updateOneInDatabase(database, appsRuntimeState, { _id: kept }, { $set: merged }); + // eslint-disable-next-line no-await-in-loop + await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier, _id: { $ne: kept } }); } - await database.collection(appsRuntimeState).createIndex({ identifier: 1 }, { unique: true, name: 'identifier_unique' }); - } catch (err) { - log.error(`appsRuntimeState - failed to prepare collection: ${err.message}`); } + await database.collection(appsRuntimeState).createIndex({ identifier: 1 }, { unique: true, name: 'identifier_unique' }); } module.exports = { diff --git a/tests/unit/appsRuntimeState.test.js b/tests/unit/appsRuntimeState.test.js index 29eeefa3d3..a911ef5f0f 100644 --- a/tests/unit/appsRuntimeState.test.js +++ b/tests/unit/appsRuntimeState.test.js @@ -1030,10 +1030,20 @@ describe('appsRuntimeState tests', () => { ops.length = 0; writeFails = true; - await orderState.prepareCollection(); + const error = await orderState.prepareCollection().catch((caught) => caught); + expect(error, 'boot is told the collection is not prepared').to.be.an('error'); expect(ops, 'a twin was removed although the merge was never written').to.deep.equal([]); }); + it('rejects when the unique index cannot be built, so boot prepares the collection again', async () => { + docs = []; + createIndexStub.rejects(new Error('connection timed out')); + + const error = await prepState.prepareCollection().catch((caught) => caught); + + expect(error.message).to.equal('connection timed out'); + }); + it('carries fields the merge does not name, from whichever twin has them', async () => { docs = [ { From d09bb6a922652dbadfb787fe18fe554062e91ce3 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:36:33 +0100 Subject: [PATCH 063/369] fix(db): boot drops a superseded index only when it is there dropIndexIfPresent settles when the index or its collection does not exist and rejects on any other failure, so a database that did not answer fails boot's schema step instead of leaving the superseded TTL indexes in place. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/serviceManager.js | 30 ++++++++++++++++++++------ tests/unit/serviceManager.test.js | 25 ++++++++++++++++++++- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index ab1acdc08b..0b4122e3af 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -215,6 +215,23 @@ async function ensureIndex(collection, spec, options = {}, recover = null) { } } +/** + * Drops the index `name` from `collection` when it is there. Rejects on any + * failure other than the index or its collection not existing, so boot + * prepares the schema again. + * @param {object} collection - a mongo collection handle + * @param {string} name - the index name + * @returns {Promise} + */ +async function dropIndexIfPresent(collection, name) { + try { + await collection.dropIndex(name); + } catch (error) { + if (error?.codeName === 'IndexNotFound' || error?.codeName === 'NamespaceNotFound') return; + throw error; + } +} + /** * Assert every index one collection needs, in a single command. * @@ -402,7 +419,7 @@ async function startFluxFunctions() { ]); await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); // TTL is driven by expireAt (set per-document by store functions). Migrate from old broadcastedAt-based TTL. - await databaseTemp.collection(config.database.appsglobal.collections.appsLocations).dropIndex('broadcastedAt_1').catch(() => {}); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, @@ -420,22 +437,22 @@ async function startFluxFunctions() { { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, ]); log.info('App state events collection prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts).dropIndex('broadcastedAt_1').catch(() => {}); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { broadcastedAt: 1 } }, { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, ]); log.info('Signed appinstalling broadcasts collection prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations).dropIndex('broadcastedAt_1').catch(() => {}); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, ]); log.info('Flux Apps installing locations prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).dropIndex('cachedAt_1').catch(() => {}); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).dropIndex('broadcastedAt_1').catch(() => {}); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'cachedAt_1'); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, @@ -443,7 +460,7 @@ async function startFluxFunctions() { { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, ]); log.info('App installing errors locations prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts).dropIndex('broadcastedAt_1').catch(() => {}); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { broadcastedAt: 1 } }, @@ -960,6 +977,7 @@ module.exports = { startFluxFunctions, ensureIndex, ensureIndexes, + dropIndexIfPresent, dedupeByKey, // testing exports dosState, diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index 3dc913a157..791164a1fa 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -14,7 +14,7 @@ chai.use(chaiAsPromised); const { expect } = chai; const { - ensureIndex, ensureIndexes, dedupeByKey, dosState, + ensureIndex, ensureIndexes, dropIndexIfPresent, dedupeByKey, dosState, } = require('../../ZelBack/src/services/serviceManager'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const log = require('../../ZelBack/src/lib/log'); @@ -129,6 +129,29 @@ describe('serviceManager ensureIndex', () => { }); }); +describe('serviceManager dropIndexIfPresent', () => { + const serverError = (code, codeName) => Object.assign(new Error(codeName), { code, codeName }); + + it('drops the index', async () => { + const dropIndex = sinon.stub().resolves(); + + await dropIndexIfPresent({ dropIndex }, 'broadcastedAt_1'); + + sinon.assert.calledOnceWithExactly(dropIndex, 'broadcastedAt_1'); + }); + + it('settles when the index or its collection is not there', async () => { + await dropIndexIfPresent({ dropIndex: sinon.stub().rejects(serverError(27, 'IndexNotFound')) }, 'broadcastedAt_1'); + await dropIndexIfPresent({ dropIndex: sinon.stub().rejects(serverError(26, 'NamespaceNotFound')) }, 'broadcastedAt_1'); + }); + + it('rejects when the database does not answer, so boot prepares the schema again', async () => { + const timedOut = Object.assign(new Error('connection timed out'), { name: 'MongoNetworkTimeoutError' }); + + await expect(dropIndexIfPresent({ dropIndex: sinon.stub().rejects(timedOut) }, 'broadcastedAt_1')).to.be.rejectedWith('timed out'); + }); +}); + describe('serviceManager dedupeByKey', () => { function stubCollection(groups) { return { From f0d0cf4fbd64bea8d37a3088a9fc45eb73510113 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:39:18 +0100 Subject: [PATCH 064/369] fix(db): the registry check reports valid only when it was checked isReindexAppsInformationRequired rejects when the database does not answer and answers null when the messages give no live app to validate against. validateAppsInformation reports validated:false in both cases, with the error when there is one. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 102 ++++++++++++++--------------- tests/unit/dbHelper.test.js | 9 +++ tests/unit/registryRebuild.test.js | 19 ++++++ 3 files changed, 79 insertions(+), 51 deletions(-) diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index d9c346649b..c0fc07fd10 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -707,13 +707,14 @@ async function liveAppRow(appsGlobalDb, appsMessagesCol, daemonDb, name, scanned /** * Whether the registry holds a different number of live apps than the messages give: each name's * newest message, and for a name the name rule decides (appNamesTheNameRuleDecides), its message - * in force (liveAppRow), as the rebuild takes them. + * in force (liveAppRow), as the rebuild takes them. Null when the messages give no live app, so + * there is nothing to validate against. Rejects when the database does not answer. * @param {mongodb.Db} appsGlobalDb * @param {string} appsMessagesCol mongo collection name * @param {string} appsInformationCol mongo collection name * @param {number} scannedHeight * @param {mongodb.Db} daemonDb the payment records' database - * @returns {Promise} + * @returns {Promise} */ async function isReindexAppsInformationRequired( appsGlobalDb, @@ -773,61 +774,56 @@ async function isReindexAppsInformationRequired( }, ]; - try { - await ensureNewestAppMessageIndex(appsGlobalDb.collection(appsMessagesCol)); - - const messagesCursor = await aggregateInDatabase( - appsGlobalDb, - appsMessagesCol, - appsMessagesPipeline, - { returnArray: false }, - ); - const informationCursor = await aggregateInDatabase( - appsGlobalDb, - appsInformationCol, - appsInformationPipeline, - { returnArray: false }, - ); + await ensureNewestAppMessageIndex(appsGlobalDb.collection(appsMessagesCol)); - const newestLive = await messagesCursor.next(); - const appsFromInformation = await informationCursor.next(); - const appsFromMessagesCount = (newestLive?.count ?? 0) + decidedLive; + const messagesCursor = await aggregateInDatabase( + appsGlobalDb, + appsMessagesCol, + appsMessagesPipeline, + { returnArray: false }, + ); + const informationCursor = await aggregateInDatabase( + appsGlobalDb, + appsInformationCol, + appsInformationPipeline, + { returnArray: false }, + ); - if (!appsFromMessagesCount) { - log.warn('No apps from apps messages found, unable to validate apps information'); - return false; - } + const newestLive = await messagesCursor.next(); + const appsFromInformation = await informationCursor.next(); + const appsFromMessagesCount = (newestLive?.count ?? 0) + decidedLive; - if (!appsFromInformation) { - log.info('No apps information apps found, reindexing colleciton'); - return true; - } + if (!appsFromMessagesCount) { + log.warn('No apps from apps messages found, unable to validate apps information'); + return null; + } - log.info( - `Apps reindex validation. Found ${appsFromMessagesCount} apps from appsMessages.` - + ` Found ${appsFromInformation.count} apps from appsInformation`, - ); + if (!appsFromInformation) { + log.info('No apps information apps found, reindexing colleciton'); + return true; + } - if (appsFromMessagesCount !== appsFromInformation.count) { - return true; - } + log.info( + `Apps reindex validation. Found ${appsFromMessagesCount} apps from appsMessages.` + + ` Found ${appsFromInformation.count} apps from appsInformation`, + ); - // Detect ghost flat fields on v4+ specs caused by $set accumulating - // fields from prior spec versions. Fixed by replaceOne in registryManager. - const ghostCount = await countInDatabase(appsGlobalDb, appsInformationCol, { - version: { $gte: 4 }, - repotag: { $exists: true }, - }); - if (ghostCount > 0) { - log.info(`Found ${ghostCount} v4+ specs with ghost fields from prior versions, reindex required`); - return true; - } + if (appsFromMessagesCount !== appsFromInformation.count) { + return true; + } - return false; - } catch (err) { - log.error(`isReindexAppsInformationRequired - Mongodb Error: ${err}`); - return false; + // Detect ghost flat fields on v4+ specs caused by $set accumulating + // fields from prior spec versions. Fixed by replaceOne in registryManager. + const ghostCount = await countInDatabase(appsGlobalDb, appsInformationCol, { + version: { $gte: 4 }, + repotag: { $exists: true }, + }); + if (ghostCount > 0) { + log.info(`Found ${ghostCount} v4+ specs with ghost fields from prior versions, reindex required`); + return true; } + + return false; } /** @@ -1016,8 +1012,9 @@ async function reindexGlobalAppsInformation( /** * Verifies the app count based on an aggregation from appsmessages and compares it to the * app count in appsinformation. If they differ - the appsinformation collection is dropped and - * rebuilt from the appsmessages. The entire process takes about 500-700ms. - * @returns {Promise<{validated: boolean, reindexed: boolean}>} + * rebuilt from the appsmessages. The entire process takes about 500-700ms. Not validated, with + * the error, when the check could not be made. + * @returns {Promise<{validated: boolean, reindexed: boolean, error?: string}>} */ async function validateAppsInformation() { const response = { validated: false, reindexed: false }; @@ -1074,6 +1071,8 @@ async function validateAppsInformation() { log.info(`validateAppsInformation reindexRequired: ${reindexRequired}`); + if (reindexRequired === null) return response; + if (!reindexRequired) { response.validated = true; return response; @@ -1092,6 +1091,7 @@ async function validateAppsInformation() { response.reindexed = true; } catch (err) { log.error(`Unable to validate apps information. Error: ${err}`); + response.error = err.message; } return response; } diff --git a/tests/unit/dbHelper.test.js b/tests/unit/dbHelper.test.js index f08d795b02..9cf731a3ec 100644 --- a/tests/unit/dbHelper.test.js +++ b/tests/unit/dbHelper.test.js @@ -836,6 +836,15 @@ describe('dbHelper tests', () => { expect(apps.map((a) => `${a.name}:${a.hash}`).sort()).to.deep.equal(['IncidentApp:i4', 'LegacyApp:l3', 'RenewedApp:r2', 'TiedApp:ta']); }); + it('does not report a registry valid when there is nothing to validate it against', async () => { + await appsGlobalDb.collection(appsglobal.collections.appsMessages).deleteMany({}); + + const result = await dbHelper.validateAppsInformation(); + + expect(result.validated).to.equal(false); + expect(result.reindexed).to.equal(false); + }); + it('should judge the rebuilt collection as valid, not reindex it on every check', async () => { const first = await dbHelper.validateAppsInformation(); expect(first.reindexed).to.equal(true); diff --git a/tests/unit/registryRebuild.test.js b/tests/unit/registryRebuild.test.js index d8e55ac7e2..a256cc83af 100644 --- a/tests/unit/registryRebuild.test.js +++ b/tests/unit/registryRebuild.test.js @@ -11,6 +11,7 @@ const fs = require('fs'); const path = require('path'); const { expect } = require('chai'); +const { MongoNetworkTimeoutError } = require('mongodb'); const espree = require('espree'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); @@ -276,6 +277,24 @@ describe('registry rebuild: who holds a name', () => { expect(await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb)).to.equal(true); }); + it('rejects when the registry cannot be read, rather than reading it as consistent', async () => { + await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); + await rebuild(); + const unreadable = { + collection: (name) => (name === INFO + ? { aggregate: () => ({ next: async () => { throw new MongoNetworkTimeoutError('timed out'); } }) } + : globalDb.collection(name)), + }; + + const error = await dbHelper.isReindexAppsInformationRequired(unreadable, MESSAGES, INFO, HEIGHT, daemonDb).catch((caught) => caught); + + expect(error).to.be.instanceOf(MongoNetworkTimeoutError); + }); + + it('answers null, not consistent, when the messages give no live app to validate against', async () => { + expect(await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb)).to.equal(null); + }); + it('removes this node\'s install when the newest message is another owner\'s live registration and the holder\'s app has expired', async () => { await seed( [msg({ hash: 'holder', owner: 'alice', height: 1000, expire: 1500 }), msg({ hash: 'taker', owner: 'mallory', height: 2000 })], From 2b9fedd24ccc2c9b9f8743cec928f44b03e4d392 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:40:48 +0100 Subject: [PATCH 065/369] fix(db): a login is written only by the request that removed its phrase verifyLogin removes the login phrase before it writes the login, and refuses the login when the phrase cannot be removed or another request removed it first. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/idService.js | 28 +++++++++++++++----------- tests/unit/idService.test.js | 33 +++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 12 deletions(-) diff --git a/ZelBack/src/services/idService.js b/ZelBack/src/services/idService.js index 365262108d..d59cb6abbe 100644 --- a/ZelBack/src/services/idService.js +++ b/ZelBack/src/services/idService.js @@ -26,17 +26,17 @@ const PRIVILEGE_RESPONSE = Object.freeze({ NONE: 'none', }); -async function deleteLoginPhrase(phrase) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.local.database); - const collection = config.database.local.collections.activeLoginPhrases; - const query = { loginPhrase: phrase }; - const projection = {}; - await dbHelper.findOneAndDeleteInDatabase(database, collection, query, projection); - } catch (error) { - log.error(error); - } +/** + * Removes a login phrase, so that it signs in once. + * @param {string} phrase + * @returns {Promise} Whether this call removed it. Rejects when the + * database does not answer. + */ +async function consumeLoginPhrase(phrase) { + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.local.database); + const collection = config.database.local.collections.activeLoginPhrases; + return Boolean(await dbHelper.findOneAndDeleteInDatabase(database, collection, { loginPhrase: phrase }, {})); } @@ -196,6 +196,11 @@ async function verifyLogin(req, res) { } else if (address === adminZelid) { privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } + // A phrase signs in once: the login is written only by the request that + // removed it. + if (!await consumeLoginPhrase(message)) { + throw new Error('Signed message is no longer valid. Please request a new one.'); + } // The signature is the session's credential. The login websocket hands it // to whoever holds the login phrase, so it is kept apart from the login, // in a collection whose TTL index expires it a minute after the login. It @@ -219,7 +224,6 @@ async function verifyLogin(req, res) { }; const resMessage = messageHelper.createDataMessage(resData); res.json(resMessage); - deleteLoginPhrase(message); } else { throw new Error('Invalid signature'); } diff --git a/tests/unit/idService.test.js b/tests/unit/idService.test.js index b32eaed203..c34033ddbc 100644 --- a/tests/unit/idService.test.js +++ b/tests/unit/idService.test.js @@ -668,6 +668,7 @@ describe('idService tests', () => { loginPhrase: `${timestamp - 10000}11111111111111111111111111111`, }); sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({ loginPhrase: 'removed' }); await dbHelper.initiateDB(); dbHelper.databaseConnection(); const req = { @@ -696,12 +697,44 @@ describe('idService tests', () => { sinon.assert.calledOnceWithMatch(res.json, expectedError); }); + const loginWithPhrase = async (removed) => { + bitcoinMessageStub.returns(true); + const timestamp = Date.now(); + const message = `${timestamp - 300000}11111111111111111111111111111`; + sinon.stub(dbHelper, 'findOneInDatabase').resolves({ loginPhrase: message }); + const insertStub = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').callsFake(removed); + await dbHelper.initiateDB(); + const mockStream = new PassThrough(); + mockStream.push(JSON.stringify({ zelid: '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg', signature: '1234356asdf', message })); + mockStream.end(); + const res = generateResponse(); + await idService.verifyLogin(mockStream, res); + await serviceHelper.delay(100); + return { res, insertStub }; + }; + + it('writes no login when its phrase cannot be removed', async () => { + const { res, insertStub } = await loginWithPhrase(async () => { throw new Error('connection timed out'); }); + + sinon.assert.notCalled(insertStub); + sinon.assert.calledOnceWithMatch(res.json, { status: 'error' }); + }); + + it('writes no login when another request removed its phrase first', async () => { + const { res, insertStub } = await loginWithPhrase(async () => null); + + sinon.assert.notCalled(insertStub); + sinon.assert.calledOnceWithMatch(res.json, { status: 'error', data: { message: 'Signed message is no longer valid. Please request a new one.' } }); + }); + it('should keep the signature off the login, in its own collection, written first and with no timer', async () => { const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); bitcoinMessageStub.returns(true); const timestamp = Date.now(); const message = `${timestamp - 300000}11111111111111111111111111111`; sinon.stub(dbHelper, 'findOneInDatabase').resolves({ loginPhrase: message }); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({ loginPhrase: message }); const insertStub = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); await dbHelper.initiateDB(); dbHelper.databaseConnection(); From 672cba454c69596c360d81e3aea92c1622724473 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:42:24 +0100 Subject: [PATCH 066/369] fix(db): a registry row that cannot be read is not taken for an absent app getStrictApplicationSpecifications rejects when the registry cannot be read and answers null only when the app has no row. The component redeploys then answer the database error, the obsolete-app pass stops and runs again on its schedule, and the enterprise check on a non-ArcaneOS node runs again after 5 min while a database read stops it. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 47 ++++++++++++------- tests/unit/advancedWorkflows.test.js | 33 +++++++++++++ 2 files changed, 63 insertions(+), 17 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 0b655db5e1..98754aeaa7 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -295,24 +295,23 @@ function hasV8ComponentStructureChange(appSpecifications, installedApp) { return componentCountChanged || componentNamesChanged; } -// Get strict application specifications +/** + * The registry row of `appName`, or null when it has none. Rejects when the + * registry cannot be read. + * @param {string} appName + * @returns {Promise} + */ async function getStrictApplicationSpecifications(appName) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); - const query = { name: appName }; - const projection = { - projection: { - _id: 0, - }, - }; - const appInfo = await dbHelper.findOneInDatabase(database, globalAppsInformation, query, projection); - return appInfo; - } catch (error) { - log.error(`Error getting strict app specifications for ${appName}:`, error); - return null; - } + const query = { name: appName }; + const projection = { + projection: { + _id: 0, + }, + }; + return dbHelper.findOneInDatabase(database, globalAppsInformation, query, projection); } /** @@ -468,12 +467,17 @@ async function findAndRestoreNonEnterpriseSpecs(installedApp) { return specsForRemoval; } +// How long after an enterprise check a database read stopped it runs again. +const ENTERPRISE_CHECK_RETRY_MS = 5 * 60 * 1000; + /** * Check and remove enterprise apps (v8+) running on non-arcaneOS nodes. - * This function runs once at startup to clean up incompatible apps. + * This function runs once at startup to clean up incompatible apps, and again + * after ENTERPRISE_CHECK_RETRY_MS while a database read stops it. * @returns {Promise} Completion status */ async function checkAndRemoveEnterpriseAppsOnNonArcane() { + let unread = false; try { // Skip if running on arcaneOS if (isArcane) { @@ -487,6 +491,7 @@ async function checkAndRemoveEnterpriseAppsOnNonArcane() { const installedAppsRes = await getInstalledAppsFromDb(); if (installedAppsRes.status !== 'success') { log.error('Failed to get installed apps for enterprise check'); + unread = true; return; } @@ -538,6 +543,7 @@ async function checkAndRemoveEnterpriseAppsOnNonArcane() { } } catch (error) { log.error(`Error processing app ${installedApp.name} for enterprise check:`, error); + if (dbHelper.isDatabaseError(error)) unread = true; // Continue with next app even if this one fails } } @@ -545,6 +551,13 @@ async function checkAndRemoveEnterpriseAppsOnNonArcane() { log.info('Enterprise app compatibility check completed'); } catch (error) { log.error('Error in checkAndRemoveEnterpriseAppsOnNonArcane:', error); + if (dbHelper.isDatabaseError(error)) unread = true; + } finally { + if (unread) { + log.warn(`Enterprise app compatibility check could not read every app, running again in ${ENTERPRISE_CHECK_RETRY_MS / 60000} min`); + const timer = setTimeout(checkAndRemoveEnterpriseAppsOnNonArcane, ENTERPRISE_CHECK_RETRY_MS); + if (timer.unref) timer.unref(); + } } } diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 72e95c7b91..59a1eba86d 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -314,6 +314,28 @@ describe('advancedWorkflows tests', () => { }); }); + describe('checkAndRemoveEnterpriseAppsOnNonArcane', () => { + afterEach(() => sinon.restore()); + + it('runs again once the registry it could not read answers, and not after', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findInDatabase').resolves([{ name: 'enterpriseCandidate' }]); + const registryRead = sinon.stub(dbHelper, 'findOneInDatabase'); + registryRead.onFirstCall().rejects(new MongoNetworkTimeoutError('timed out')); + registryRead.resolves({ name: 'enterpriseCandidate', version: 7 }); + + await advancedWorkflows.checkAndRemoveEnterpriseAppsOnNonArcane(); + sinon.assert.calledOnce(registryRead); + + await clock.tickAsync(5 * 60 * 1000); + sinon.assert.calledTwice(registryRead); + + await clock.tickAsync(60 * 60 * 1000); + sinon.assert.calledTwice(registryRead); + }); + }); + describe('softRedeployComponent tests', () => { let globalState; let res; @@ -340,6 +362,17 @@ describe('advancedWorkflows tests', () => { resetGlobalState(); }); + it('answers the database error, not "not found", when the registry cannot be read, and clears its flag', async () => { + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findOneInDatabase').rejects(new MongoNetworkTimeoutError('timed out')); + + const error = await advancedWorkflows.softRedeployComponent('myapp', 'frontend', res).catch((caught) => caught); + + expect(error).to.be.instanceOf(MongoNetworkTimeoutError); + expect(globalState.softRedeployInProgress).to.equal(false); + sinon.restore(); + }); + it('should return early if removal is in progress', async () => { globalState.removalInProgress = true; From e0afef190e7b04a05e173a30bc848c85092b5794 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:44:19 +0100 Subject: [PATCH 067/369] fix(db): an orphan removal whose location cannot be read is broadcast forceAppRemovals broadcasts the removal of an orphan when this node's location row for it cannot be read: a removal of a row that does not exist changes nothing on a peer. The pass removes nothing unless both the apps docker runs and the installed apps were read. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 12 +++-- tests/unit/advancedWorkflows.test.js | 48 +++++++++++++++++++ 2 files changed, 57 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 98754aeaa7..a6cb5f01f1 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -4796,8 +4796,13 @@ async function forceAppRemovals() { } const dockerAppsReported = await appQueryService.listAllApps(); - const dockerApps = dockerAppsReported.data; const installedAppsRes = await appQueryService.installedApps(); + // An app is an orphan only against a list of installed apps that was read. + if (dockerAppsReported.status !== 'success' || installedAppsRes.status !== 'success') { + log.warn('Unable to read the apps docker runs and the installed apps, skipping forceAppRemovals'); + return; + } + const dockerApps = dockerAppsReported.data; const appsInstalled = installedAppsRes.data; const dockerAppsNames = dockerApps.map((app) => { if (app.Names[0].startsWith('/zel')) { @@ -4838,8 +4843,9 @@ async function forceAppRemovals() { log.info(`${dApp} not found in locations for this IP (${localSocketAddr}), skipping broadcast`); } } catch (locationError) { - log.error(`Error checking app location for ${dApp}: ${locationError.message}`); - // Default to not broadcasting on error to avoid false positives + // A removal of a row that does not exist changes nothing on a peer. + shouldBroadcast = true; + log.error(`Error checking app location for ${dApp}, will broadcast removal: ${locationError.message}`); } // eslint-disable-next-line no-await-in-loop diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 59a1eba86d..490e332c30 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -32,6 +32,10 @@ const registryManager = require('../../ZelBack/src/services/appDatabase/registry const appInspector = require('../../ZelBack/src/services/appManagement/appInspector'); const volumeService = require('../../ZelBack/src/services/utils/volumeService'); const dockerServiceModule = require('../../ZelBack/src/services/dockerService'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); +const appQueryService = require('../../ZelBack/src/services/appQuery/appQueryService'); +const appUninstaller = require('../../ZelBack/src/services/appLifecycle/appUninstaller'); // The operator stop lock as appsRuntimeState.operatorStopState answers it. const UNLOCKED = { stopped: false, startRequested: false, force: false }; @@ -314,6 +318,50 @@ describe('advancedWorkflows tests', () => { }); }); + describe('forceAppRemovals', () => { + let remove; + + beforeEach(() => { + resetGlobalState(); + sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('203.0.113.60:16127'); + sinon.stub(appQueryService, 'listAllApps').resolves({ status: 'success', data: [{ Names: ['/fluxweb_orphanapp'] }] }); + sinon.stub(appQueryService, 'installedApps').resolves({ status: 'success', data: [] }); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(registryManager, 'getApplicationGlobalSpecifications').resolves({ name: 'orphanapp' }); + sinon.stub(serviceHelper, 'delay').resolves(); + remove = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + }); + + afterEach(() => sinon.restore()); + + it('broadcasts the removal of an orphan whose location cannot be read', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').rejects(new MongoNetworkTimeoutError('timed out')); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.calledOnceWithExactly(remove, 'orphanapp', null, true, true, true); + }); + + it('does not broadcast the removal of an orphan this node is not listed for', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.calledOnceWithExactly(remove, 'orphanapp', null, true, true, false); + }); + + it('removes nothing when the installed apps cannot be read', async () => { + appQueryService.installedApps.resolves({ status: 'error', data: { message: 'timed out' } }); + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + const warned = sinon.stub(log, 'warn'); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.notCalled(remove); + sinon.assert.calledWithMatch(warned, /Unable to read the apps docker runs and the installed apps/); + }); + }); + describe('checkAndRemoveEnterpriseAppsOnNonArcane', () => { afterEach(() => sinon.restore()); From 6f1ef132c400ea149c9e89728fe46feb1583cd37 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:46:09 +0100 Subject: [PATCH 068/369] fix(db): a merged incident rollup is written before its duplicates are removed prepareIncidentRollup writes the merged rollup onto its newest duplicate and then removes the others, so a write that fails leaves the duplicates for the next boot to merge and never loses the incident's count. Co-Authored-By: Claude Opus 5.5 --- .../services/appTamperingDetectionService.js | 9 ++++---- .../unit/appTamperingDetectionService.test.js | 22 ++++++++++++++++--- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/ZelBack/src/services/appTamperingDetectionService.js b/ZelBack/src/services/appTamperingDetectionService.js index a80a66ff8f..966fbfb2a9 100644 --- a/ZelBack/src/services/appTamperingDetectionService.js +++ b/ZelBack/src/services/appTamperingDetectionService.js @@ -542,8 +542,7 @@ async function checkNodeReboot() { * unique index the upsert relies on is already in place. * * Low-stakes local telemetry on a 30-day TTL, so a failure logs and is left for - * the next boot rather than wedging startup - the disposition its owner chooses, - * the same one appsRuntimeState.prepareCollection takes. + * the next boot rather than wedging startup. */ async function prepareIncidentRollup() { try { @@ -582,10 +581,12 @@ async function prepareIncidentRollup() { }; delete merged._id; log.warn(`appTamperingDetection - merged ${group.length} duplicate incident rollups for ${query.appName}/${query.eventType} (count=${merged.count})`); + // The merge lands on one rollup before the others go, so a failure part way + // leaves duplicates for the next boot to merge, never the incident lost. // eslint-disable-next-line no-await-in-loop - await dbHelper.removeDocumentsFromCollection(database, tamperingEventsCollection, query); + await dbHelper.updateOneInDatabase(database, tamperingEventsCollection, { _id: newest._id }, { $set: merged }); // eslint-disable-next-line no-await-in-loop - await dbHelper.updateOneInDatabase(database, tamperingEventsCollection, query, { $set: merged }, { upsert: true }); + await dbHelper.removeDocumentsFromCollection(database, tamperingEventsCollection, { ...query, _id: { $ne: newest._id } }); } await database.collection(tamperingEventsCollection).createIndex( diff --git a/tests/unit/appTamperingDetectionService.test.js b/tests/unit/appTamperingDetectionService.test.js index b60a828e1d..ebdf7bffda 100644 --- a/tests/unit/appTamperingDetectionService.test.js +++ b/tests/unit/appTamperingDetectionService.test.js @@ -787,16 +787,20 @@ describe('appTamperingDetectionService tests', () => { it('sums counts and keeps the widest window when merging duplicate rollups', async () => { findResults = [ { - appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 2, firstSeen: new Date(2000), lastSeen: new Date(5000), severity: 3, + _id: 'older', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 2, firstSeen: new Date(2000), lastSeen: new Date(5000), severity: 3, }, { - appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 5, firstSeen: new Date(1000), lastSeen: new Date(9000), severity: 3, + _id: 'newer', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 5, firstSeen: new Date(1000), lastSeen: new Date(9000), severity: 3, }, ]; await service.prepareIncidentRollup(); - expect(removed).to.deep.equal([{ appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1' }]); + expect(upserts.map(({ query }) => query)).to.deep.equal([{ _id: 'newer' }]); + expect(removed).to.deep.equal([{ + appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', _id: { $ne: 'newer' }, + }]); + sinon.assert.callOrder(dbHelperStub.updateOneInDatabase, dbHelperStub.removeDocumentsFromCollection); expect(upserts).to.have.lengthOf(1); const merged = upserts[0].set; expect(merged.count, 'counts were not summed').to.equal(7); @@ -830,6 +834,18 @@ describe('appTamperingDetectionService tests', () => { sinon.assert.calledOnce(createIndexStub); }); + it('removes no duplicate when the merged rollup cannot be written', async () => { + findResults = [ + { _id: 'a', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 2 }, + { _id: 'b', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 5 }, + ]; + dbHelperStub.updateOneInDatabase = sinon.stub().rejects(new Error('connection timed out')); + + await service.prepareIncidentRollup(); + + expect(removed).to.deep.equal([]); + }); + it('logs and does not throw when the merge fails, leaving it for the next boot', async () => { dbHelperStub.findInDatabase = sinon.stub().rejects(new Error('mongo mid-election')); From 31f6bfaab10d3394c542264234fe6ba280f60ed0 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:47:43 +0100 Subject: [PATCH 069/369] fix(db): a boot that could not be recorded is recorded once the database answers checkNodeReboot records the boot in the boot history and the startup marker, and runs that again after 30 s, 60 s, 5 min and then every 15 min until it succeeds. A boot already in the history is not pushed again, so a retry after a marker write that failed records it once. Co-Authored-By: Claude Opus 5.5 --- .../services/appTamperingDetectionService.js | 87 ++++++++++++++----- .../unit/appTamperingDetectionService.test.js | 34 +++++++- 2 files changed, 96 insertions(+), 25 deletions(-) diff --git a/ZelBack/src/services/appTamperingDetectionService.js b/ZelBack/src/services/appTamperingDetectionService.js index 966fbfb2a9..0bd8a7c6b0 100644 --- a/ZelBack/src/services/appTamperingDetectionService.js +++ b/ZelBack/src/services/appTamperingDetectionService.js @@ -490,20 +490,43 @@ async function checkNodeReboot() { currentBootId = bootId; const now = new Date(); - const previous = await dbHelper.findOneInDatabase( - database, nodeStartupTrackerCollection, { _id: STARTUP_MARKER_KEY }, + // The machine's boot moment (uptime-derived): cadence consumers want the + // boot period itself, free of FluxOS's variable start lag. + const bootedAt = new Date(now.getTime() - Math.round(os.uptime() * 1000)); + await recordBootUntilWritten(now, bootedAt); + } catch (error) { + log.error(`appTamperingDetection - checkNodeReboot failed: ${error.message}`); + } +} + +// The delays before a boot that could not be recorded is recorded again, by +// attempt; the last repeats until it is. +const BOOT_RECORD_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; + +/** + * Records the boot this process runs in: in the boot history the first time + * its boot_id is seen there, and as the startup marker. Repeating it records + * nothing twice. + * @param {Date} now When FluxOS first saw this boot. + * @param {Date} bootedAt When the machine booted. + * @returns {Promise} + */ +async function recordBoot(now, bootedAt) { + const database = dbHelper.databaseConnection().db(config.database.local.database); + const previous = await dbHelper.findOneInDatabase( + database, nodeStartupTrackerCollection, { _id: STARTUP_MARKER_KEY }, + ); + const previousBootId = previous && previous.bootId ? previous.bootId : null; + + if (previousBootId !== currentBootId) { + // First sighting of this boot_id: genuine reboot, first run, or upgrade + // from the pre-boot_id marker. All three land in the boot history; a + // same-boot_id process restart does not, keeping the history a record + // of machine boots rather than FluxOS restarts. + const history = await dbHelper.findOneInDatabase( + database, nodeStartupTrackerCollection, { _id: BOOT_HISTORY_KEY }, ); - const previousBootId = previous && previous.bootId ? previous.bootId : null; - - if (previousBootId !== currentBootId) { - // First sighting of this boot_id: genuine reboot, first run, or upgrade - // from the pre-boot_id marker. All three land in the boot history; a - // same-boot_id process restart does not, keeping the history a record - // of machine boots rather than FluxOS restarts. `bootedAt` is the - // machine's boot moment (uptime-derived) — cadence consumers want the - // boot period itself, free of FluxOS's variable start lag; `at` is when - // FluxOS first saw the boot. - const bootedAt = new Date(now.getTime() - Math.round(os.uptime() * 1000)); + if (!(history?.boots ?? []).some((boot) => boot.bootId === currentBootId)) { await dbHelper.findOneAndUpdateInDatabase( database, nodeStartupTrackerCollection, @@ -511,20 +534,36 @@ async function checkNodeReboot() { { $push: { boots: { $each: [{ bootId: currentBootId, bootedAt, at: now }], $slice: -BOOT_HISTORY_MAX } } }, { upsert: true }, ); - if (previousBootId) { - log.info(`appTamperingDetection - machine reboot: boot_id ${previousBootId.slice(0, 8)} -> ${currentBootId.slice(0, 8)}`); - } } + if (previousBootId) { + log.info(`appTamperingDetection - machine reboot: boot_id ${previousBootId.slice(0, 8)} -> ${currentBootId.slice(0, 8)}`); + } + } - await dbHelper.findOneAndUpdateInDatabase( - database, - nodeStartupTrackerCollection, - { _id: STARTUP_MARKER_KEY }, - { $set: { at: now, bootId: currentBootId } }, - { upsert: true }, - ); + await dbHelper.findOneAndUpdateInDatabase( + database, + nodeStartupTrackerCollection, + { _id: STARTUP_MARKER_KEY }, + { $set: { at: now, bootId: currentBootId } }, + { upsert: true }, + ); +} + +/** + * recordBoot, run again after BOOT_RECORD_RETRY_MS until it succeeds. + * @param {Date} now + * @param {Date} bootedAt + * @param {number} [attempt] + * @returns {Promise} + */ +async function recordBootUntilWritten(now, bootedAt, attempt = 0) { + try { + await recordBoot(now, bootedAt); } catch (error) { - log.error(`appTamperingDetection - checkNodeReboot failed: ${error.message}`); + const delay = BOOT_RECORD_RETRY_MS[Math.min(attempt, BOOT_RECORD_RETRY_MS.length - 1)]; + log.error(`appTamperingDetection - this boot could not be recorded, retrying in ${Math.round(delay / 1000)}s: ${error.message}`); + const timer = setTimeout(recordBootUntilWritten, delay, now, bootedAt, attempt + 1); + if (timer.unref) timer.unref(); } } diff --git a/tests/unit/appTamperingDetectionService.test.js b/tests/unit/appTamperingDetectionService.test.js index ebdf7bffda..d1411a967c 100644 --- a/tests/unit/appTamperingDetectionService.test.js +++ b/tests/unit/appTamperingDetectionService.test.js @@ -755,13 +755,45 @@ describe('appTamperingDetectionService tests', () => { expect(dbHelperStub.removeDocumentsFromCollection.called).to.be.false; }); - it('swallows errors without throwing and logs them', async () => { + it('records the boot once the database answers, without throwing meanwhile', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); dbHelperStub.findOneInDatabase = sinon.stub().rejects(new Error('mongo down')); await service.checkNodeReboot(); sinon.assert.calledOnce(logStub.error); expect(logStub.error.firstCall.args[0]).to.include('mongo down'); + expect(historyUpdates()).to.have.lengthOf(0); + + setMarker({ _id: 'lastStartup', at: new Date(), bootId: PREVIOUS_BOOT_ID }); + await clock.tickAsync(30 * 1000); + + expect(historyUpdates()).to.have.lengthOf(1); + expect(markerUpdates()).to.have.lengthOf(1); + clock.restore(); + }); + + it('does not record a boot twice when the marker write after it failed', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + const history = { _id: 'bootHistory', boots: [] }; + dbHelperStub.findOneInDatabase = sinon.stub().callsFake(async (db, coll, query) => { + if (query._id === 'lastStartup') return { _id: 'lastStartup', bootId: PREVIOUS_BOOT_ID }; + if (query._id === 'bootHistory') return history; + return null; + }); + const write = dbHelperStub.findOneAndUpdateInDatabase; + dbHelperStub.findOneAndUpdateInDatabase = sinon.stub().callsFake(async (db, coll, query, update, options) => { + if (query._id === 'bootHistory') history.boots.push(...update.$push.boots.$each); + if (query._id === 'lastStartup' && dbHelperStub.findOneAndUpdateInDatabase.callCount === 2) throw new Error('mongo down'); + return write(db, coll, query, update, options); + }); + + await service.checkNodeReboot(); + await clock.tickAsync(30 * 1000); + + expect(history.boots.map((boot) => boot.bootId)).to.deep.equal([CURRENT_BOOT_ID]); + expect(markerUpdates()).to.have.lengthOf(1); + clock.restore(); }); }); From 6178fb0e888b253a2ee603818a1f6a39b71e60d8 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:49:10 +0100 Subject: [PATCH 070/369] fix(db): a tampering event that could not be written is written later recordEvent keeps an observation it could not write in memory, at most 500, and writes the waiting ones in order after 30 s, 60 s, 5 min and then every 15 min until they are written, each under the time it was seen. The app's attribution is part of the write: a read of it that fails fails the write, so no incident is recorded without its owner and spec hash. Co-Authored-By: Claude Opus 5.5 --- .../services/appTamperingDetectionService.js | 239 +++++++++++------- .../unit/appTamperingDetectionService.test.js | 48 +++- 2 files changed, 192 insertions(+), 95 deletions(-) diff --git a/ZelBack/src/services/appTamperingDetectionService.js b/ZelBack/src/services/appTamperingDetectionService.js index 0bd8a7c6b0..36cb6583d9 100644 --- a/ZelBack/src/services/appTamperingDetectionService.js +++ b/ZelBack/src/services/appTamperingDetectionService.js @@ -268,7 +268,8 @@ function deriveMainAppName(appName) { * Owner/spec-hash attribution for an app from the local installed-apps * database, cached per emitted name. Tries the name as passed first (a real * app name may itself start with 'flux'), then the derived main app name. - * Best-effort: returns null fields when the app cannot be found. + * Null fields when the app cannot be found; rejects when the installed apps + * cannot be read. * @param {string} appName - Name exactly as passed by the emitter * @returns {Promise} */ @@ -276,33 +277,132 @@ async function getAppAttribution(appName) { if (!appName) return null; const cached = appAttributionCache.get(appName); if (cached && Date.now() - cached.cachedAt < ATTRIBUTION_TTL_MS) return cached; + const db = dbHelper.databaseConnection(); + if (!db) throw new Error('no database connection'); + const appsDatabase = db.db(config.database.appslocal.database); + const projection = { projection: { _id: 0, owner: 1, hash: 1 } }; + let app = await dbHelper.findOneInDatabase( + appsDatabase, localAppsInformation, { name: appName }, projection, + ); + if (!app) { + const mainName = deriveMainAppName(appName); + if (mainName !== appName) { + app = await dbHelper.findOneInDatabase( + appsDatabase, localAppsInformation, { name: mainName }, projection, + ); + } + } + const attribution = { + ownerZelid: app?.owner ?? null, + appHash: app?.hash ?? null, + cachedAt: Date.now(), + }; + appAttributionCache.set(appName, attribution); + return attribution; +} + +// The delays before incidents that could not be written are written again, by +// attempt; the last repeats until they are. At most PENDING_INCIDENTS_MAX wait, +// the oldest going first. A retry is idempotent for the score: it sums one +// severity per incident document, whatever its count. +const INCIDENT_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; +const PENDING_INCIDENTS_MAX = 500; +const pendingIncidents = []; +let incidentRetryTimer = null; + +/** + * Writes one observation onto its incident rollup. Rejects when the database + * does not answer. + * @param {object} observation From recordEvent. + * @returns {Promise} + */ +async function writeIncident(observation) { + const { + appName, eventType, details, at, bootId, uptimeSec, + } = observation; + const db = dbHelper.databaseConnection(); + if (!db) throw new Error('no database connection'); + const database = db.db(config.database.local.database); + const incidentKey = `${bootId ?? 'unknown'}:${Math.floor(at.getTime() / INCIDENT_BUCKET_MS)}`; + const [identity, attribution] = await Promise.all([ + getNodeIdentity(), + getAppAttribution(appName), + ]); + const query = { appName, eventType, incidentKey }; + const update = { + $setOnInsert: { + schemaVersion: 1, + appName, + eventType, + incidentKey, + severity: EVENT_SEVERITY[eventType] ?? 0, + bootId, + uptimeSecAtEvent: uptimeSec, + firstSeen: at, + detailsSample: details, + nodeTxid: identity?.nodeTxid ?? null, + nodeOutidx: identity?.nodeOutidx ?? null, + nodeIp: identity?.nodeIp ?? null, + pubkey: identity?.pubkey ?? null, + paymentAddress: identity?.paymentAddress ?? null, + ownerZelid: attribution?.ownerZelid ?? null, + appHash: attribution?.appHash ?? null, + }, + $set: { lastSeen: at }, + $inc: { count: 1 }, + }; + let result; try { - const db = dbHelper.databaseConnection(); - if (!db) return null; - const appsDatabase = db.db(config.database.appslocal.database); - const projection = { projection: { _id: 0, owner: 1, hash: 1 } }; - let app = await dbHelper.findOneInDatabase( - appsDatabase, localAppsInformation, { name: appName }, projection, + result = await dbHelper.findOneAndUpdateInDatabase( + database, tamperingEventsCollection, query, update, { upsert: true }, ); - if (!app) { - const mainName = deriveMainAppName(appName); - if (mainName !== appName) { - app = await dbHelper.findOneInDatabase( - appsDatabase, localAppsInformation, { name: mainName }, projection, - ); - } - } - const attribution = { - ownerZelid: app?.owner ?? null, - appHash: app?.hash ?? null, - cachedAt: Date.now(), - }; - appAttributionCache.set(appName, attribution); - return attribution; } catch (error) { - log.debug(`appTamperingDetection - attribution lookup failed for ${appName}: ${error.message}`); - return null; + // Two concurrent upserts racing on a brand-new incident key can trip + // the unique index; the loser retries once and lands as the increment. + if (error && error.code === 11000) { + result = await dbHelper.findOneAndUpdateInDatabase( + database, tamperingEventsCollection, query, update, { upsert: true }, + ); + } else { + throw error; + } + } + // Insert detection across driver result shapes: v6 returns the pre-image + // doc or null; older shapes return { value, lastErrorObject }. + const inserted = !result || result.value === null || result?.lastErrorObject?.updatedExisting === false; + if (inserted) { + log.info(`appTamperingDetection - recorded ${eventType} for ${appName}`); + } else { + log.debug(`appTamperingDetection - ${eventType} for ${appName} repeated, incident count incremented`); + } +} + +/** + * Writes the pending incidents in order, stopping at the first that fails and + * trying again after the next delay. + * @param {number} attempt How many flushes failed before this one. + * @returns {Promise} + */ +async function flushPendingIncidents(attempt) { + incidentRetryTimer = null; + while (pendingIncidents.length) { + try { + // eslint-disable-next-line no-await-in-loop + await writeIncident(pendingIncidents[0]); + pendingIncidents.shift(); + } catch (error) { + scheduleIncidentRetry(attempt + 1); + return; + } } + log.info('appTamperingDetection - pending incidents written'); +} + +function scheduleIncidentRetry(attempt) { + if (incidentRetryTimer) return; + const delay = INCIDENT_RETRY_MS[Math.min(attempt, INCIDENT_RETRY_MS.length - 1)]; + incidentRetryTimer = setTimeout(flushPendingIncidents, delay, attempt); + if (incidentRetryTimer.unref) incidentRetryTimer.unref(); } /** @@ -314,80 +414,37 @@ async function getAppAttribution(appName) { * identity, app attribution and severity so fleet-level consumers can * attribute and weigh it without joining local state. * Documents expire 30 days after lastSeen (TTL index). + * An event that cannot be written waits in memory and is written later + * (INCIDENT_RETRY_MS), under the time it was seen. * @param {string} appName - Application name * @param {string} eventType - One of the EVENT_SEVERITY keys * @param {string} details - Free-text context (stored once per incident) */ async function recordEvent(rawAppName, eventType, details) { + // Stored under the app's own name whatever a caller addressed it by. The + // boot sweep walks docker component identifiers and the reconciler works + // in app names, so without this the same fault on the same app lands in + // two rows and getEvents, which matches the name exactly, finds one of + // them. Idempotent: an app name carries neither the prefix nor the + // component part this strips. + const observation = { + appName: deriveMainAppName(rawAppName), + eventType, + details, + at: new Date(), + bootId: currentBootId, + uptimeSec: Math.round(os.uptime()), + }; try { - const db = dbHelper.databaseConnection(); - if (!db) { - log.warn('appTamperingDetection - DB not available, skipping event'); - return; - } - const database = db.db(config.database.local.database); - const now = new Date(); - const incidentKey = `${currentBootId ?? 'unknown'}:${Math.floor(now.getTime() / INCIDENT_BUCKET_MS)}`; - // Stored under the app's own name whatever a caller addressed it by. The - // boot sweep walks docker component identifiers and the reconciler works - // in app names, so without this the same fault on the same app lands in - // two rows and getEvents, which matches the name exactly, finds one of - // them. Idempotent: an app name carries neither the prefix nor the - // component part this strips. - const appName = deriveMainAppName(rawAppName); - const [identity, attribution] = await Promise.all([ - getNodeIdentity(), - getAppAttribution(appName), - ]); - const query = { appName, eventType, incidentKey }; - const update = { - $setOnInsert: { - schemaVersion: 1, - appName, - eventType, - incidentKey, - severity: EVENT_SEVERITY[eventType] ?? 0, - bootId: currentBootId, - uptimeSecAtEvent: Math.round(os.uptime()), - firstSeen: now, - detailsSample: details, - nodeTxid: identity?.nodeTxid ?? null, - nodeOutidx: identity?.nodeOutidx ?? null, - nodeIp: identity?.nodeIp ?? null, - pubkey: identity?.pubkey ?? null, - paymentAddress: identity?.paymentAddress ?? null, - ownerZelid: attribution?.ownerZelid ?? null, - appHash: attribution?.appHash ?? null, - }, - $set: { lastSeen: now }, - $inc: { count: 1 }, - }; - let result; - try { - result = await dbHelper.findOneAndUpdateInDatabase( - database, tamperingEventsCollection, query, update, { upsert: true }, - ); - } catch (error) { - // Two concurrent upserts racing on a brand-new incident key can trip - // the unique index; the loser retries once and lands as the increment. - if (error && error.code === 11000) { - result = await dbHelper.findOneAndUpdateInDatabase( - database, tamperingEventsCollection, query, update, { upsert: true }, - ); - } else { - throw error; - } - } - // Insert detection across driver result shapes: v6 returns the pre-image - // doc or null; older shapes return { value, lastErrorObject }. - const inserted = !result || result.value === null || result?.lastErrorObject?.updatedExisting === false; - if (inserted) { - log.info(`appTamperingDetection - recorded ${eventType} for ${appName}`); - } else { - log.debug(`appTamperingDetection - ${eventType} for ${appName} repeated, incident count incremented`); - } + await writeIncident(observation); } catch (error) { - log.error(`appTamperingDetection - failed to record event: ${error.message}`); + log.error(`appTamperingDetection - failed to record event, written again later: ${error.message}`); + if (pendingIncidents.length >= PENDING_INCIDENTS_MAX) { + const dropped = pendingIncidents.shift(); + log.warn(`appTamperingDetection - ${PENDING_INCIDENTS_MAX} incidents wait to be written, dropping ${dropped.eventType} for ${dropped.appName}`); + } + pendingIncidents.push(observation); + scheduleIncidentRetry(0); } } diff --git a/tests/unit/appTamperingDetectionService.test.js b/tests/unit/appTamperingDetectionService.test.js index d1411a967c..6d70035a51 100644 --- a/tests/unit/appTamperingDetectionService.test.js +++ b/tests/unit/appTamperingDetectionService.test.js @@ -446,21 +446,61 @@ describe('appTamperingDetectionService tests', () => { expect(logStub.debug.calledWithMatch(/repeated, incident count incremented/)).to.be.true; }); - it('no-ops when DB is not available', async () => { + it('writes an event once the database is available, under the time it was seen', async () => { + const clock = sinon.useFakeTimers({ now: new Date('2026-07-16T10:59:50Z'), toFake: ['setTimeout', 'Date'] }); + const connection = dbHelperStub.databaseConnection; dbHelperStub.databaseConnection = sinon.stub().returns(null); await service.recordEvent('myapp', 'container_vanished', 'x'); - expect(dbHelperStub.findOneAndUpdateInDatabase.called).to.be.false; + + dbHelperStub.databaseConnection = connection; + await clock.tickAsync(30 * 1000); + + expect(eventUpserts()).to.have.lengthOf(1); + const { query, update } = eventUpserts()[0]; + expect(query.incidentKey).to.equal(`unknown:${Math.floor(Date.parse('2026-07-16T10:59:50Z') / service.INCIDENT_BUCKET_MS)}`); + expect(update.$setOnInsert.firstSeen).to.deep.equal(new Date('2026-07-16T10:59:50Z')); + clock.restore(); }); - it('swallows write errors without throwing and logs them', async () => { + it('writes events that failed again, with a growing delay, until they are written', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + const write = dbHelperStub.findOneAndUpdateInDatabase; dbHelperStub.findOneAndUpdateInDatabase = sinon.stub().rejects(new Error('boom')); await service.recordEvent('myapp', 'container_vanished', 'x'); + await service.recordEvent('otherapp', 'mount_vanished', 'y'); - sinon.assert.calledOnce(logStub.error); + sinon.assert.calledTwice(logStub.error); expect(logStub.error.firstCall.args[0]).to.include('boom'); + + await clock.tickAsync(30 * 1000); + sinon.assert.calledThrice(dbHelperStub.findOneAndUpdateInDatabase); + await clock.tickAsync(30 * 1000); + sinon.assert.calledThrice(dbHelperStub.findOneAndUpdateInDatabase); + + dbHelperStub.findOneAndUpdateInDatabase = write; + await clock.tickAsync(30 * 1000); + + expect(eventUpserts().map((c) => c.query.appName)).to.deep.equal(['myapp', 'otherapp']); + clock.restore(); + }); + + it('does not write an incident without its owner when the installed apps cannot be read', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + installedApps.myapp = { owner: 'ownerZ', hash: 'hashH' }; + const read = dbHelperStub.findOneInDatabase; + dbHelperStub.findOneInDatabase = sinon.stub().rejects(new Error('timed out')); + + await service.recordEvent('myapp', 'container_vanished', 'x'); + expect(eventUpserts()).to.have.lengthOf(0); + + dbHelperStub.findOneInDatabase = read; + await clock.tickAsync(30 * 1000); + + expect(eventUpserts()[0].update.$setOnInsert.ownerZelid).to.equal('ownerZ'); + clock.restore(); }); }); From a1e72a55b1d35c309e4ca7ff84d1b3416168f28a Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:50:16 +0100 Subject: [PATCH 071/369] refactor(db): registryManager carries no getters without callers getAppSpecificationFromDb, getAllAppsInformation, getRunningApps and getAllGlobalApplications have no callers and are removed with their tests. Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 84 ------------------- tests/unit/registryManager.test.js | 58 ------------- 2 files changed, 142 deletions(-) diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index 073cf928a2..7c2755f35f 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -1611,41 +1611,6 @@ async function checkApplicationRegistrationNameConflicts(appSpecFormatted, hash) return true; } -/** - * Get app specification from database - * @param {string} appName - Application name - * @returns {Promise} App specification - */ -async function getAppSpecificationFromDb(appName) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - - const query = { name: new RegExp(`^${appName}$`, 'i') }; - const projection = { projection: { _id: 0 } }; - - const appSpec = await dbHelper.findOneInDatabase(database, globalAppsInformation, query, projection); - return appSpec; - } catch (error) { - log.error(`Error getting app specification from database: ${error.message}`); - return null; - } -} - -/** - * Get all apps information (both global and local) - * @returns {Promise} Array of all app information - */ -async function getAllAppsInformation() { - try { - const allApps = await availableApps(); - return allApps; - } catch (error) { - log.error(`Error getting all apps information: ${error.message}`); - return []; - } -} - /** * Get installed apps information * @returns {Promise} Array of installed apps. Rejects when the read fails: @@ -1658,26 +1623,6 @@ async function getInstalledApps() { return dbHelper.findInDatabase(localDatabase, localAppsInformation, {}, projection); } -/** - * Get running apps information - * @returns {Promise} Array of running apps - */ -async function getRunningApps() { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - - const query = {}; - const projection = { projection: { _id: 0 } }; - - const runningApps = await dbHelper.findInDatabase(database, globalAppsLocations, query, projection); - return runningApps; - } catch (error) { - log.error(`Error getting running apps: ${error.message}`); - return []; - } -} - /** * To get all apps running on a specific IP address. Returns all apps running on this ip * @param {string} ip IP address to check @@ -1726,31 +1671,6 @@ function registrationInformation(_req, res) { } } -/** - * Get all global applications from database - * @param {string[]} proj - Optional projection fields - * @returns {Promise} Array of global applications - */ -async function getAllGlobalApplications(proj = []) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - const query = {}; - const wantedProjection = { - _id: 0, - }; - proj.forEach((field) => { - wantedProjection[field] = 1; - }); - const projection = { projection: wantedProjection, sort: { height: 1 } }; // ensure sort from oldest to newest - const results = await dbHelper.findInDatabase(database, globalAppsInformation, query, projection); - return results; - } catch (error) { - log.error(error); - return []; - } -} - /** * Remove expired applications from global database and local installations * @returns {Promise} Completion status @@ -2607,13 +2527,9 @@ module.exports = { storeAppSpecificationInForce, governingAppMessage, isGoverningAppMessage, - getAppSpecificationFromDb, - getAllAppsInformation, getInstalledApps, - getRunningApps, getRunningAppIpList, registrationInformation, - getAllGlobalApplications, expireGlobalApplications, reindexGlobalAppsInformation, reindexGlobalAppsLocation, diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 4a1332db9c..b8ff17527e 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1598,56 +1598,6 @@ describe('registryManager tests', () => { }); }); - describe('getAllGlobalApplications tests', () => { - beforeEach(async () => { - const collection = config.database.appsglobal.collections.appsInformation; - const apps = [ - { - name: 'App1', - owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', - height: 100, - }, - { - name: 'App2', - owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', - height: 200, - }, - ]; - - try { - await database.collection(collection).drop(); - } catch (err) { - // Collection doesn't exist - } - await dbHelper.insertManyToDatabase(database, collection, apps); - }); - - it('should return all global applications', async () => { - const result = await registryManager.getAllGlobalApplications(); - - expect(result).to.be.an('array'); - expect(result.length).to.be.at.least(2); - }); - - it('should return applications with specific projections', async () => { - const result = await registryManager.getAllGlobalApplications(['name', 'owner']); - - expect(result).to.be.an('array'); - if (result.length > 0) { - expect(result[0]).to.have.property('name'); - expect(result[0]).to.have.property('owner'); - } - }); - - it('should sort applications by height', async () => { - const result = await registryManager.getAllGlobalApplications(['name', 'height']); - - if (result.length > 1) { - expect(result[0].height).to.be.at.most(result[1].height); - } - }); - }); - describe('getInstalledApps tests', () => { it('should return installed apps from local database', async () => { const result = await registryManager.getInstalledApps(); @@ -1665,14 +1615,6 @@ describe('registryManager tests', () => { }); }); - describe('getRunningApps tests', () => { - it('should return running apps from global locations', async () => { - const result = await registryManager.getRunningApps(); - - expect(result).to.be.an('array'); - }); - }); - describe('getRunningAppIpList tests', () => { beforeEach(async () => { const collection = config.database.appsglobal.collections.appsLocations; From dafaf487a7b0f92dab222abf8ec5bdf8c57160be Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:53:59 +0100 Subject: [PATCH 072/369] fix(db): boot holds at a database step that fails instead of starting again prepareDatabaseSchema holds every index, merge and repair boot runs on the database, and boot runs it, and the payment position backfill, again every 15 s until each succeeds. A database that does not answer then holds boot at that step, and nothing boot already started (intervals, subscriptions, manageAppsOnBoot, container crash recovery) runs a second time. Clearing signatures from login rows is one of the schema steps, and its failure fails the step. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/serviceManager.js | 305 ++++++++++++++----------- tests/unit/serviceManager.test.js | 44 +++- 2 files changed, 206 insertions(+), 143 deletions(-) diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 0b4122e3af..a93991498a 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -6,6 +6,7 @@ const https = require('https'); const cacheManager = require('./utils/cacheManager').default; const log = require('../lib/log'); const dbHelper = require('./dbHelper'); +const serviceHelper = require('./serviceHelper'); const explorerService = require('./explorerService'); const fluxCommunication = require('./fluxCommunication'); const networkStateService = require('./networkStateService'); @@ -165,10 +166,10 @@ async function dedupeByKey(collection, spec, options = {}) { * index rather than running degraded without it; * - anything else rethrows. * - * The rethrow is deliberate and is NOT the blanket swallow it replaced. Index - * setup runs before any service or interval starts, so the 15s startFluxFunctions - * retry re-runs it safely: a TRANSIENT failure (mongo mid-election, a slow-disk - * blip) heals on the next pass instead of being skipped until the next reboot, + * The rethrow is deliberate. Index setup runs in prepareDatabaseSchema, which + * boot runs again every 15 s until it succeeds (untilSucceeds): a TRANSIENT + * failure (mongo mid-election, a slow-disk blip) heals on the next pass instead + * of being skipped until the next reboot, * and a genuinely UNRECOVERABLE database wedges loudly - which is correct, since * a node whose DB cannot hold its schema cannot serve apps and appremove would * not rescue it. The realistic wedge that finding motivated - a unique index @@ -273,6 +274,166 @@ async function ensureIndexes(collection, specs) { } } +// How long boot waits before it runs a database step that failed again. +const BOOT_STEP_RETRY_MS = 15 * 1000; + +/** + * Runs `step` until it succeeds, waiting BOOT_STEP_RETRY_MS after each + * failure. Boot holds at a database step that fails rather than starting again + * from the top, so nothing boot already started runs twice. + * @param {string} what Names the step in the log. + * @param {function(): Promise<*>} step + * @returns {Promise<*>} What the step resolved. + */ +async function untilSucceeds(what, step) { + try { + return await step(); + } catch (error) { + log.error(`${what} failed, running it again in ${BOOT_STEP_RETRY_MS / 1000}s: ${error.message}`); + await serviceHelper.delay(BOOT_STEP_RETRY_MS); + return untilSucceeds(what, step); + } +} + +/** + * Brings every collection boot depends on to its schema: indexes, the merges + * their unique indexes need, and the one-off repairs. Every step is safe to + * repeat, and any failure rejects. + * @returns {Promise} + */ +async function prepareDatabaseSchema() { + log.info('Preparing local database...'); + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.local.database); + // Logins survive a restart: every request re-verifies its signature and + // re-checks its privilege against the node's current config, and each + // session collection's TTL index expires its rows. A login row never holds + // the session's signature (loginSignatures does, for a minute); one that + // does loses it here. + await dbHelper.updateInDatabase(database, config.database.local.collections.loggedUsers, { signature: { $exists: true } }, { $unset: { signature: '' } }); + // Named literally because they are no longer part of the schema: the payment + // request and receipt collections outlived the endpoint that wrote them, and + // a node's word was never the payment record - the chain is. Dropping is + // idempotent, so this costs one 'ns not found' per boot once they are gone. + await Promise.all(['activepaymentrequests', 'completedpayments'].map((orphan) => dbHelper + .dropCollection(database, orphan).catch((error) => { + if (error.message !== 'ns not found') { + log.error(error); + } + }))); + await ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ + { key: { createdAt: 1 }, expireAfterSeconds: 14 * 24 * 60 * 60 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ + { key: { createdAt: 1 }, expireAfterSeconds: 900 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ + { key: { createdAt: 1 }, expireAfterSeconds: 900 }, + ]); + await ensureIndexes(database.collection(config.database.local.collections.loginSignatures), [ + { key: { createdAt: 1 }, expireAfterSeconds: 60 }, + { key: { loginPhrase: 1 } }, + ]); + // legacy pre-incident-schema rows expire via detectedAt; current incident + // documents expire via lastSeen. The tamper service purges pre-schema + // rows at startup, so the detectedAt pair only matters where old code + // still writes; drop it once the fleet is past the incident schema. + await ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ + { key: { detectedAt: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days + { key: { appName: 1, detectedAt: -1 }, name: 'appName_detectedAt' }, + { key: { lastSeen: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days + { key: { appName: 1, eventType: 1, lastSeen: -1 }, name: 'appName_eventType_lastSeen' }, + ]); + // The unique incident-rollup index lives with its owner: duplicate rollups + // must have their counts SUMMED, not one dropped, so the merge needs the + // collection's own knowledge rather than a generic dedupe. See + // prepareIncidentRollup. + await appTamperingDetectionService.prepareIncidentRollup(); + await appTamperingDetectionService.checkNodeReboot(); + // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, + // then enforce one doc per component identifier + await appsRuntimeState.prepareCollection(); + log.info('Local database prepared'); + log.info('Preparing temporary database...'); + // no need to drop temporary messages + const databaseTemp = db.db(config.database.appsglobal.database); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ + { key: { receivedAt: 1 }, expireAfterSeconds: tempMsgTtlS }, + ]); + log.info('Temporary database prepared'); + log.info('Preparing Flux Apps locations'); + + // ToDo: Fix all these broken database drops / index creations / removals all over the place. The prior dropIndex was removing the + // index entirely so there was no index at all! + + // The below index is created in the Explorer Service. We need to remove all the database indexing from the Explorer Service. + // It's not the explorer service's responsibility, and other services need these indexes before Explorer Service creates them. + + // It should be the dbService's responsibility that the db is in a state fit for use. + + // we have to create this index again here, as we need it to repair the db. As we were deleting this on every reboot (and it was only created when scannedHeight was 0) + // Creating an index that already exists is a no-op + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ + { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dedupeByKey }, + { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, + { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, + ]); + await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); + // TTL is driven by expireAt (set per-document by store functions). Migrate from old broadcastedAt-based TTL. + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), 'broadcastedAt_1'); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, + { key: { ip: 1, name: 1 } }, + ]); + log.info('Flux Apps locations prepared'); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dedupeByKey }, + { key: { broadcastedAt: 1 } }, + { key: { createdAt: 1 } }, + // the events of one type still standing, every move among them + { key: { type: 1, expireAt: 1 }, name: 'events by type and expiry' }, + // the moves that lead to an address + { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, + ]); + log.info('App state events collection prepared'); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), 'broadcastedAt_1'); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { broadcastedAt: 1 } }, + { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, + ]); + log.info('Signed appinstalling broadcasts collection prepared'); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), 'broadcastedAt_1'); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, + { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, + ]); + log.info('Flux Apps installing locations prepared'); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'cachedAt_1'); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'broadcastedAt_1'); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, + { key: { name: 1, hash: 1 }, name: 'query for getting flux app install errors location based on specs name and hash' }, + { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, + ]); + log.info('App installing errors locations prepared'); + await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), 'broadcastedAt_1'); + await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { broadcastedAt: 1 } }, + { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, + ]); + log.info('Signed app installing errors broadcasts collection prepared'); + + // This fixes an issue where the appsMessage db has NaN for valueSat. Once db is repaired on all nodes, + // we can remove this. + await dbHelper.repairNanInAppsMessagesDb(); +} + /** * To start FluxOS. A series of checks are performed on port and UPnP (Universal Plug and Play) support and mapping. Database connections are established. The other relevant functions required to start FluxOS services are called. * @@ -339,138 +500,8 @@ async function startFluxFunctions() { log.info('System service initiated'); fluxadmService.start(); log.info('Fluxadm access service initiated'); - log.info('Preparing local database...'); + await untilSucceeds('Database schema preparation', prepareDatabaseSchema); const db = dbHelper.databaseConnection(); - const database = db.db(config.database.local.database); - // Logins survive a restart: every request re-verifies its signature and - // re-checks its privilege against the node's current config, and each - // session collection's TTL index expires its rows. A login row never holds - // the session's signature (loginSignatures does, for a minute); one that - // does loses it here. - await dbHelper.updateInDatabase(database, config.database.local.collections.loggedUsers, { signature: { $exists: true } }, { $unset: { signature: '' } }).catch((error) => { - log.error(error); - }); - // Named literally because they are no longer part of the schema: the payment - // request and receipt collections outlived the endpoint that wrote them, and - // a node's word was never the payment record - the chain is. Dropping is - // idempotent, so this costs one 'ns not found' per boot once they are gone. - await Promise.all(['activepaymentrequests', 'completedpayments'].map((orphan) => dbHelper - .dropCollection(database, orphan).catch((error) => { - if (error.message !== 'ns not found') { - log.error(error); - } - }))); - await ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ - { key: { createdAt: 1 }, expireAfterSeconds: 14 * 24 * 60 * 60 }, - ]); - await ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ - { key: { createdAt: 1 }, expireAfterSeconds: 900 }, - ]); - await ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ - { key: { createdAt: 1 }, expireAfterSeconds: 900 }, - ]); - await ensureIndexes(database.collection(config.database.local.collections.loginSignatures), [ - { key: { createdAt: 1 }, expireAfterSeconds: 60 }, - { key: { loginPhrase: 1 } }, - ]); - // legacy pre-incident-schema rows expire via detectedAt; current incident - // documents expire via lastSeen. The tamper service purges pre-schema - // rows at startup, so the detectedAt pair only matters where old code - // still writes; drop it once the fleet is past the incident schema. - await ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ - { key: { detectedAt: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days - { key: { appName: 1, detectedAt: -1 }, name: 'appName_detectedAt' }, - { key: { lastSeen: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days - { key: { appName: 1, eventType: 1, lastSeen: -1 }, name: 'appName_eventType_lastSeen' }, - ]); - // The unique incident-rollup index lives with its owner: duplicate rollups - // must have their counts SUMMED, not one dropped, so the merge needs the - // collection's own knowledge rather than a generic dedupe. See - // prepareIncidentRollup. - await appTamperingDetectionService.prepareIncidentRollup(); - await appTamperingDetectionService.checkNodeReboot(); - // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, - // then enforce one doc per component identifier - await appsRuntimeState.prepareCollection(); - log.info('Local database prepared'); - log.info('Preparing temporary database...'); - // no need to drop temporary messages - const databaseTemp = db.db(config.database.appsglobal.database); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ - { key: { receivedAt: 1 }, expireAfterSeconds: tempMsgTtlS }, - ]); - log.info('Temporary database prepared'); - log.info('Preparing Flux Apps locations'); - - // ToDo: Fix all these broken database drops / index creations / removals all over the place. The prior dropIndex was removing the - // index entirely so there was no index at all! - - // The below index is created in the Explorer Service. We need to remove all the database indexing from the Explorer Service. - // It's not the explorer service's responsibility, and other services need these indexes before Explorer Service creates them. - - // It should be the dbService's responsibility that the db is in a state fit for use. - - // we have to create this index again here, as we need it to repair the db. As we were deleting this on every reboot (and it was only created when scannedHeight was 0) - // Creating an index that already exists is a no-op - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ - { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dedupeByKey }, - { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, - { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, - ]); - await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); - // TTL is driven by expireAt (set per-document by store functions). Migrate from old broadcastedAt-based TTL. - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), 'broadcastedAt_1'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, - { key: { ip: 1, name: 1 } }, - ]); - log.info('Flux Apps locations prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dedupeByKey }, - { key: { broadcastedAt: 1 } }, - { key: { createdAt: 1 } }, - // the events of one type still standing, every move among them - { key: { type: 1, expireAt: 1 }, name: 'events by type and expiry' }, - // the moves that lead to an address - { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, - ]); - log.info('App state events collection prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), 'broadcastedAt_1'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { broadcastedAt: 1 } }, - { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, - ]); - log.info('Signed appinstalling broadcasts collection prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), 'broadcastedAt_1'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, - { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, - ]); - log.info('Flux Apps installing locations prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'cachedAt_1'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'broadcastedAt_1'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, - { key: { name: 1, hash: 1 }, name: 'query for getting flux app install errors location based on specs name and hash' }, - { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, - ]); - log.info('App installing errors locations prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), 'broadcastedAt_1'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { broadcastedAt: 1 } }, - { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, - ]); - log.info('Signed app installing errors broadcasts collection prepared'); - - // This fixes an issue where the appsMessage db has NaN for valueSat. Once db is repaired on all nodes, - // we can remove this. - await dbHelper.repairNanInAppsMessagesDb(); // The location table this node already holds, brought back as soon as the // database is up. Detached and best-effort - every consumer degrades safely @@ -556,7 +587,7 @@ async function startFluxFunctions() { log.error('fluxd is running without addressindex=1. FluxOS needs the address index to order app messages. Set addressindex=1 in flux.conf and reindex the daemon, or reinstall the node. FluxOS is stopping.'); process.exit(1); } - await paymentPositions.backfillPaymentPositions(); + await untilSucceeds('Payment position backfill', () => paymentPositions.backfillPaymentPositions()); // Initialize app sync orchestrator and spawner const orchestrator = new AppSyncOrchestrator({ @@ -978,6 +1009,8 @@ module.exports = { ensureIndex, ensureIndexes, dropIndexIfPresent, + untilSucceeds, + prepareDatabaseSchema, dedupeByKey, // testing exports dosState, diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index 791164a1fa..7061a4918c 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -1,9 +1,8 @@ -// ensureIndex is the boot sequence's self-healing index builder. It runs in the -// awaited stretch of startFluxFunctions, before any service or interval starts, -// so a throw re-runs boot safely: the recoverable failures (an options conflict, -// or a unique build blocked by duplicate rows) are healed here, and anything -// else rethrows - a transient blip heals on the next boot pass, a genuinely -// broken DB wedges loudly. dedupeByKey is the recovery strategy a unique build +// ensureIndex is the boot sequence's self-healing index builder. It runs in +// prepareDatabaseSchema, which boot runs again until it succeeds: the +// recoverable failures (an options conflict, or a unique build blocked by +// duplicate rows) are healed here, and anything else rethrows - a transient +// blip heals on the next pass, a genuinely broken DB wedges loudly. dedupeByKey is the recovery strategy a unique build // passes when its key is the row's identity. const chai = require('chai'); @@ -14,8 +13,10 @@ chai.use(chaiAsPromised); const { expect } = chai; const { - ensureIndex, ensureIndexes, dropIndexIfPresent, dedupeByKey, dosState, + ensureIndex, ensureIndexes, dropIndexIfPresent, untilSucceeds, prepareDatabaseSchema, dedupeByKey, dosState, } = require('../../ZelBack/src/services/serviceManager'); +const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const log = require('../../ZelBack/src/lib/log'); @@ -152,6 +153,35 @@ describe('serviceManager dropIndexIfPresent', () => { }); }); +describe('serviceManager untilSucceeds', () => { + afterEach(() => sinon.restore()); + + it('runs a failing step again after a wait, until it succeeds, and answers what it resolved', async () => { + const wait = sinon.stub(serviceHelper, 'delay').resolves(); + const step = sinon.stub(); + step.onFirstCall().rejects(new Error('connection timed out')); + step.onSecondCall().rejects(new Error('connection timed out')); + step.resolves('prepared'); + + expect(await untilSucceeds('a step', step)).to.equal('prepared'); + + sinon.assert.calledThrice(step); + sinon.assert.calledTwice(wait); + sinon.assert.alwaysCalledWithExactly(wait, 15 * 1000); + }); +}); + +describe('serviceManager prepareDatabaseSchema', () => { + afterEach(() => sinon.restore()); + + it('rejects when the login rows cannot be cleared of signatures, so boot prepares the schema again', async () => { + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({ collection: () => ({}) }) }); + sinon.stub(dbHelper, 'updateInDatabase').rejects(new Error('connection timed out')); + + await expect(prepareDatabaseSchema()).to.be.rejectedWith('connection timed out'); + }); +}); + describe('serviceManager dedupeByKey', () => { function stubCollection(groups) { return { From 24e3843f029ced3d02a1b96699614c7053c827fb Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:54:46 +0100 Subject: [PATCH 073/369] perf(db): the payment position backfill writes its records 5,000 at a time Each batch is one bulkWrite, so no single round trip carries every record the backfill positions. A run that stops part way leaves the rest without a position, and the next run takes them. Co-Authored-By: Claude Opus 5.5 --- .../src/services/migrations/paymentPositions.js | 13 +++++++++++-- tests/unit/paymentPositions.test.js | 14 +++++++++++++- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/migrations/paymentPositions.js b/ZelBack/src/services/migrations/paymentPositions.js index 26b29171c9..f0b6b6b409 100644 --- a/ZelBack/src/services/migrations/paymentPositions.js +++ b/ZelBack/src/services/migrations/paymentPositions.js @@ -6,6 +6,10 @@ const daemonServiceUtils = require('../daemonService/daemonServiceUtils'); const messageVerifier = require('../appMessaging/messageVerifier'); const { appPaymentPositions } = require('../appPaymentPositions'); +// Records written per round trip: each batch is one bulkWrite, so a fleet's +// worth of records never rides on a single call. +const BACKFILL_BATCH = 5000; + /** * Gives every app payment record without a position in its block the position * and the height the daemon's address index holds for its transaction: one call @@ -21,6 +25,8 @@ const { appPaymentPositions } = require('../appPaymentPositions'); * them again at the next start; the registry row of its app is written after * them, since which message governs reads their positions * (messageVerifier.moveStoredMessagesToPayments, storeGoverningAppSpecifications). + * The records are written BACKFILL_BATCH at a time; a run that stops part way + * leaves the rest without a position, and the next run takes them. * @returns {Promise<{positioned: number, heightsCorrected: number, notOnChain: number, aboveTip: number}>} */ async function backfillPaymentPositions() { @@ -72,7 +78,10 @@ async function backfillPaymentPositions() { } } const moved = await messageVerifier.moveStoredMessagesToPayments(corrected); - if (operations.length) await dbHelper.bulkWriteInDatabase(database, collection, operations); + for (let i = 0; i < operations.length; i += BACKFILL_BATCH) { + // eslint-disable-next-line no-await-in-loop + await dbHelper.bulkWriteInDatabase(database, collection, operations.slice(i, i + BACKFILL_BATCH)); + } await messageVerifier.storeGoverningAppSpecifications(moved); log.info(`migrations - payment positions: ${operations.length - notOnChain} positioned, ${heightsCorrected} of them at the chain's height rather than the recorded one, ${notOnChain} not on the chain, ${aboveTip} above the daemon's tip`); return { @@ -80,4 +89,4 @@ async function backfillPaymentPositions() { }; } -module.exports = { backfillPaymentPositions }; +module.exports = { backfillPaymentPositions, BACKFILL_BATCH }; diff --git a/tests/unit/paymentPositions.test.js b/tests/unit/paymentPositions.test.js index 809c7a8248..023afd6043 100644 --- a/tests/unit/paymentPositions.test.js +++ b/tests/unit/paymentPositions.test.js @@ -11,7 +11,7 @@ const dbHelper = require('../../ZelBack/src/services/dbHelper'); const daemonServiceUtils = require('../../ZelBack/src/services/daemonService/daemonServiceUtils'); const messageVerifier = require('../../ZelBack/src/services/appMessaging/messageVerifier'); const { appPaymentAddresses, appPaymentPositions } = require('../../ZelBack/src/services/appPaymentPositions'); -const { backfillPaymentPositions } = require('../../ZelBack/src/services/migrations/paymentPositions'); +const { backfillPaymentPositions, BACKFILL_BATCH } = require('../../ZelBack/src/services/migrations/paymentPositions'); chai.use(chaiAsPromised); const { expect } = chai; @@ -213,6 +213,18 @@ describe('app payment positions', () => { }); }); + it('writes the records in batches, each of at most BACKFILL_BATCH', async () => { + const count = (2 * BACKFILL_BATCH) + 1; + const records = Array.from({ length: count }, (_, i) => ({ txid: `t${i}`, height: 500 })); + findInDatabase.resolves(records); + executeCall.withArgs('getaddressdeltas').resolves({ status: 'success', data: records.map(({ txid }, i) => delta(txid, 500, i)) }); + + const outcome = await backfillPaymentPositions(); + + expect(bulkWriteInDatabase.getCalls().map((call) => call.args[2].length)).to.deep.equal([BACKFILL_BATCH, BACKFILL_BATCH, 1]); + expect(outcome.positioned).to.equal(count); + }); + it('writes only to a record that still has no position', async () => { findInDatabase.resolves([{ txid: 'a', height: 500 }]); executeCall.withArgs('getaddressdeltas').resolves({ status: 'success', data: [delta('a', 500, 0)] }); From a45425019c483b6dc9d9c1d0278b50523b7140f6 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:56:48 +0100 Subject: [PATCH 074/369] perf(db): the app message valueSat repair runs once repairNanInAppsMessagesDb leaves a marker in the node startup tracker once it completes, and later boots return without scanning the app messages. The repairMessagesCol command repairs whether or not the marker is there. A repair that fails leaves no marker, so the next boot runs it again. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 28 ++++++++++++----- tests/unit/dbHelper.test.js | 48 ++++++++++++++++++++++++++++++ tests/unit/globalconfig/default.js | 1 + 3 files changed, 70 insertions(+), 7 deletions(-) diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index c0fc07fd10..541391d191 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -526,14 +526,28 @@ async function updateValueSatInAppsMessages(brokenHashes, hashMap) { } } -async function repairNanInAppsMessagesDb() { - const brokenHashes = await findValueSatNanInAppsMessages(); - - if (!brokenHashes.length) return; +// The marker a completed valueSat repair leaves in the node startup tracker. +const VALUESAT_REPAIR_MARKER = 'appsMessagesValueSatRepaired'; - const hashMap = await findValueSatInAppsHashes(); +/** + * Gives every app message stored with a NaN valueSat the value its payment + * record holds. Runs once: a completed repair leaves a marker, and later calls + * return without scanning, unless `force` is set. + * @param {object} [options] + * @param {boolean} [options.force] Repair whether or not the marker is there. + * @returns {Promise} + */ +async function repairNanInAppsMessagesDb({ force = false } = {}) { + const local = databaseConnection().db(config.database.local.database); + const tracker = config.database.local.collections.nodeStartupTracker; + if (!force && await findOneInDatabase(local, tracker, { _id: VALUESAT_REPAIR_MARKER })) return; - await updateValueSatInAppsMessages(brokenHashes, hashMap); + const brokenHashes = await findValueSatNanInAppsMessages(); + if (brokenHashes.length) { + const hashMap = await findValueSatInAppsHashes(); + await updateValueSatInAppsMessages(brokenHashes, hashMap); + } + await updateOneInDatabase(local, tracker, { _id: VALUESAT_REPAIR_MARKER }, { $set: { repairedAt: new Date() } }, { upsert: true }); } /** @@ -1109,7 +1123,7 @@ async function main(command) { if (command === 'validateInfoCol') { await validateAppsInformation(); } else if (command === 'repairMessagesCol') { - await repairNanInAppsMessagesDb(); + await repairNanInAppsMessagesDb({ force: true }); } const client = databaseConnection(); diff --git a/tests/unit/dbHelper.test.js b/tests/unit/dbHelper.test.js index 9cf731a3ec..7ec8aa7a5b 100644 --- a/tests/unit/dbHelper.test.js +++ b/tests/unit/dbHelper.test.js @@ -778,6 +778,54 @@ describe('dbHelper tests', () => { }); }); + describe('repairNanInAppsMessagesDb', () => { + const { appsglobal, daemon, local } = config.database; + let messages; + let hashes; + let tracker; + + beforeEach(async () => { + await dbHelper.initiateDB(); + const client = dbHelper.databaseConnection(); + messages = client.db(appsglobal.database).collection(appsglobal.collections.appsMessages); + hashes = client.db(daemon.database).collection(daemon.collections.appsHashes); + tracker = client.db(local.database).collection(local.collections.nodeStartupTracker); + await messages.deleteMany({}); + await hashes.deleteMany({}); + await tracker.deleteMany({}); + await hashes.insertMany([{ hash: 'first', value: 5 }, { hash: 'later', value: 7 }]); + }); + + it('repairs once, and later calls do not scan again unless forced', async () => { + await messages.insertOne({ hash: 'first', valueSat: NaN }); + + await dbHelper.repairNanInAppsMessagesDb(); + expect((await messages.findOne({ hash: 'first' })).valueSat).to.equal(5); + expect(await tracker.findOne({ _id: 'appsMessagesValueSatRepaired' })).to.not.equal(null); + + await messages.insertOne({ hash: 'later', valueSat: NaN }); + await dbHelper.repairNanInAppsMessagesDb(); + expect((await messages.findOne({ hash: 'later' })).valueSat).to.be.NaN; + + await dbHelper.repairNanInAppsMessagesDb({ force: true }); + expect((await messages.findOne({ hash: 'later' })).valueSat).to.equal(7); + }); + + it('leaves no marker when the repair fails', async () => { + await messages.insertOne({ hash: 'first', valueSat: NaN }); + const globalDb = dbHelper.databaseConnection().db(appsglobal.database); + // the repaired value is refused, so the repair's write fails + await globalDb.command({ collMod: appsglobal.collections.appsMessages, validator: { valueSat: { $ne: 5 } } }); + try { + const error = await dbHelper.repairNanInAppsMessagesDb().catch((caught) => caught); + expect(error).to.be.an('error'); + } finally { + await globalDb.command({ collMod: appsglobal.collections.appsMessages, validator: {} }); + } + expect(await tracker.findOne({ _id: 'appsMessagesValueSatRepaired' })).to.equal(null); + }); + }); + describe('rebuilding globalAppsInformation from the message log', () => { const { appsglobal, appslocal, daemon } = config.database; const scannedHeight = 3008000; diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 4ca46c9505..4f6baa1549 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -39,6 +39,7 @@ module.exports = { geolocation: 'geolocation', policyDocuments: 'policydocuments', // last-known-good network policy documents, so an unreachable source does not drop enforcement benchmark: 'benchmark', + nodeStartupTracker: 'nodestartuptracker', ownAddress: 'ownaddress', // the address this node last adopted as its own, and the one it moved from }, }, From 43981fdc749de53bc3640da60e5dd3ea1ef525ba Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:57:56 +0100 Subject: [PATCH 075/369] perf(db): a forced removal reads only its own app's newest stored message When neither the local nor the global registry holds the app, removeAppLocally reads the app's newest stored message by name, newest height then newest timestamp, through the index on those fields, instead of reading every stored message. Co-Authored-By: Claude Opus 5.5 --- .../services/appLifecycle/appUninstaller.js | 22 +++++++------------ tests/unit/appUninstaller.test.js | 18 +++++++++++++++ 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 23d566f793..f5a2777b53 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -953,21 +953,15 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // get it from locally available Specifications const allApps = await availableApps(); appSpecifications = allApps.find((a) => a.name === appName); - // get it from permanent messages + // get it from permanent messages: the app's newest, read through the + // index on name, height and timestamp if (!appSpecifications) { - const query = {}; - const projection = { projection: { _id: 0 } }; - const messages = await dbHelper.findInDatabase(database, globalAppsMessages, query, projection); - const appMessages = messages.filter((message) => { - const specifications = message.appSpecifications; - return specifications.name === appName; - }); - let currentSpecifications; - appMessages.forEach((message) => { - if (!currentSpecifications || message.height > currentSpecifications.height) { - currentSpecifications = message; - } - }); + const [currentSpecifications] = await dbHelper.findInDatabase( + database, + globalAppsMessages, + { 'appSpecifications.name': appName }, + { projection: { _id: 0 }, sort: { height: -1, timestamp: -1 }, limit: 1 }, + ); if (currentSpecifications && currentSpecifications.height) { ({ appSpecifications } = currentSpecifications); } diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 88156509c3..bd4a3472f3 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -759,6 +759,24 @@ describe('appUninstaller tests', () => { }); }); + it('takes a forced removal\'s spec from the app\'s newest stored message, read by its name', async () => { + const messages = sinon.stub().resolves([{ height: 900, appSpecifications: v2Spec }]); + const uninstaller = buildUninstaller(null, {}, { findInDatabase: messages }); + const onRemoved = sinon.stub(); + uninstaller.setOnComponentRemoved(onRemoved); + + await uninstaller.removeAppLocally('testapp', res, true); + + sinon.assert.calledWithExactly( + messages, + sinon.match.any, + 'appsMessages', + { 'appSpecifications.name': 'testapp' }, + { projection: { _id: 0 }, sort: { height: -1, timestamp: -1 }, limit: 1 }, + ); + sinon.assert.calledOnceWithExactly(onRemoved, 'testapp'); + }); + it('notifies the seam with the bare name for a v1-3 app', async () => { const uninstaller = buildUninstaller(v2Spec); const onRemoved = sinon.stub(); From 14dfbe7d887ede48ef43c77e315a9d09722f9c73 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 11:59:24 +0100 Subject: [PATCH 076/369] perf(db): the hash record reconstruction reads the fields it compares and writes in batches reconstructAppMessagesHashCollection reads only the hash of each stored message and the fields of each hash record it compares, and writes its corrections 5,000 at a time. Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 21 ++++++++++++------- tests/unit/registryManager.test.js | 16 +++++++++++++- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index 7c2755f35f..439aa3c95a 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -1940,8 +1940,12 @@ async function reindexGlobalAppsInformation() { return rebuildFollowing; } +// Hash records corrected per round trip. +const HASH_RECONSTRUCTION_BATCH = 5000; + /** - * Reconstruct app messages hash collection by validating hash records against actual messages + * Reconstruct app messages hash collection by validating hash records against actual messages. + * Reads only the fields it compares, and writes its corrections HASH_RECONSTRUCTION_BATCH at a time. * @returns {Promise} Success message */ async function reconstructAppMessagesHashCollection() { @@ -1951,11 +1955,13 @@ async function reconstructAppMessagesHashCollection() { const databaseDaemon = db.db(config.database.daemon.database); const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); const currentHeight = syncStatus.data.height || 0; - const query = {}; - const projection = { projection: { _id: 0 } }; - const permanentMessages = await dbHelper.findInDatabase(databaseApps, globalAppsMessages, query, projection); - const appHashes = await dbHelper.findInDatabase(databaseDaemon, appsHashesCollection, query, projection); + const permanentMessages = await dbHelper.findInDatabase(databaseApps, globalAppsMessages, {}, { projection: { _id: 0, hash: 1 } }); + const appHashes = await dbHelper.findInDatabase(databaseDaemon, appsHashesCollection, {}, { + projection: { + _id: 0, hash: 1, txid: 1, height: 1, message: 1, messageNotFound: 1, retryFromHeight: 1, + }, + }); const permHashSet = new Set(permanentMessages.map((m) => m.hash)); const ops = []; @@ -1993,8 +1999,9 @@ async function reconstructAppMessagesHashCollection() { } let changed = 0; - if (ops.length > 0) { - const result = await databaseDaemon.collection(appsHashesCollection).bulkWrite(ops, { ordered: false }); + for (let i = 0; i < ops.length; i += HASH_RECONSTRUCTION_BATCH) { + // eslint-disable-next-line no-await-in-loop + const result = await databaseDaemon.collection(appsHashesCollection).bulkWrite(ops.slice(i, i + HASH_RECONSTRUCTION_BATCH), { ordered: false }); changed += result.modifiedCount; } diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index b8ff17527e..7b4c906b0d 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1,7 +1,7 @@ const { expect } = require('chai'); const sinon = require('sinon'); const config = require('config'); -const { ObjectId } = require('mongodb'); +const { ObjectId, Collection } = require('mongodb'); const proxyquire = require('proxyquire'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); @@ -1349,6 +1349,20 @@ describe('registryManager tests', () => { expect(unresolvedSignals()).to.equal(0); }); + it('reads only the message hashes and writes its corrections in batches', async () => { + const count = 5001; + await daemonDb.collection(appsHashes).insertMany(Array.from({ length: count }, (_, i) => record(`h${i}`, { message: false }))); + await database.collection(appsMessages).insertMany(Array.from({ length: count }, (_, i) => message(`h${i}`))); + const reads = sinon.spy(dbHelper, 'findInDatabase'); + const writes = sinon.spy(Collection.prototype, 'bulkWrite'); + + const { changed } = await registryManager.reconstructAppMessagesHashCollection(); + + expect(changed).to.equal(count); + expect(reads.firstCall.args[3]).to.deep.equal({ projection: { _id: 0, hash: 1 } }); + expect(writes.getCalls().map((call) => call.args[0].length)).to.deep.equal([5000, 1]); + }); + it('should signal from the operator route', async () => { await daemonDb.collection(appsHashes).insertOne(record('lost', { message: true })); sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); From 68dbc8b7da21a05f9519c3ec3a7b91255d68185f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:03:21 +0100 Subject: [PATCH 077/369] feat(db): the named long operations run on a maintenance client initiateDB connects a second client, maxPoolSize 4, whose socket and pool waits are bounded at 15 minutes, with the main client or not at all, and closeDbConnection closes both. Boot's schema preparation, the explorer's index builds on a fresh scan and its restore deletes, and the registry rebuild and reindex check (their index builds and $group) take their database handles from it through onMaintenanceClient. Every other call stays on the main client. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 72 +++++++++++++++++++++---- ZelBack/src/services/explorerService.js | 59 ++++++++++---------- ZelBack/src/services/serviceManager.js | 3 +- tests/unit/dbHelper.test.js | 45 +++++++++++++++- tests/unit/explorerService.test.js | 11 ++++ tests/unit/registryRebuild.test.js | 23 ++++++++ tests/unit/serviceManager.test.js | 5 +- 7 files changed, 175 insertions(+), 43 deletions(-) diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index 541391d191..04e804f019 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -19,6 +19,21 @@ const mongoUrl = `mongodb://${config.database.url}:${config.database.port}/`; */ let openDBConnection = null; +// The client for the named long operations: index builds, the explorer +// restore's deletes and the registry rebuild. One of its round trips may take +// up to MAINTENANCE_SOCKET_TIMEOUT_MS; every other call uses the main client. +const MAINTENANCE_SOCKET_TIMEOUT_MS = 15 * 60 * 1000; +const MAINTENANCE_SETTINGS = { + maxPoolSize: 4, + socketTimeoutMS: MAINTENANCE_SOCKET_TIMEOUT_MS, + waitQueueTimeoutMS: MAINTENANCE_SOCKET_TIMEOUT_MS, +}; + +/** + * @type {mongodb.MongoClient} + */ +let maintenanceDBConnection = null; + /** * Cached MongoDB server version, populated once per connection. * @type {string | null} @@ -34,6 +49,25 @@ function databaseConnection() { return openDBConnection; } +/** + * The maintenance client, if it was initiated before, otherwise null. + * @returns {mongodb.MongoClient | null} + */ +function maintenanceConnection() { + return maintenanceDBConnection; +} + +/** + * `database` on the maintenance client: the same database, whose round trips + * may take up to MAINTENANCE_SOCKET_TIMEOUT_MS. `database` itself while the + * maintenance client is not connected. + * @param {mongodb.Db} database + * @returns {mongodb.Db} + */ +function onMaintenanceClient(database) { + return maintenanceDBConnection ? maintenanceDBConnection.db(database.databaseName) : database; +} + /** * Whether an error came from the database rather than from the work around it: * a timeout, a pool wait, an unreachable server, or a refused command. A caller @@ -50,25 +84,33 @@ function isDatabaseError(error) { * Initiates connection with the database. * * @param {string} [url] + * @param {object} [settings] Client options over the main client's. * * @returns {Promise} */ -async function connectMongoDb(url) { +async function connectMongoDb(url, settings = {}) { const connectUrl = url || mongoUrl; const mongoSettings = { maxPoolSize: 100, + ...settings, }; const client = await MongoClient.connect(connectUrl, mongoSettings); return client; } /** - * Initiates default db connection. + * Initiates the default db connection and the maintenance one, both or + * neither. * @returns true */ async function initiateDB() { if (!openDBConnection) { - openDBConnection = await connectMongoDb(); + const main = await connectMongoDb(); + maintenanceDBConnection = await connectMongoDb(undefined, MAINTENANCE_SETTINGS).catch(async (error) => { + await main.close(); + throw error; + }); + openDBConnection = main; // Read the server version once, on the initial connect. It is informational // and the getter swallows its own errors, so this cannot fail the connect. await getMongoDbVersion(); @@ -135,6 +177,10 @@ async function closeDbConnection() { openDBConnection = null; mongoDbVersion = null; } + if (maintenanceDBConnection) { + await maintenanceDBConnection.close(); + maintenanceDBConnection = null; + } } /** @@ -788,10 +834,11 @@ async function isReindexAppsInformationRequired( }, ]; - await ensureNewestAppMessageIndex(appsGlobalDb.collection(appsMessagesCol)); + // The index build and the $group run on the maintenance client. + await ensureNewestAppMessageIndex(onMaintenanceClient(appsGlobalDb).collection(appsMessagesCol)); const messagesCursor = await aggregateInDatabase( - appsGlobalDb, + onMaintenanceClient(appsGlobalDb), appsMessagesCol, appsMessagesPipeline, { returnArray: false }, @@ -935,7 +982,9 @@ async function reindexGlobalAppsInformation( if (error.message !== 'ns not found') throw error; }); - const infoCol = appsGlobalDb.collection(stagingCol); + // The index builds and the $group below run on the maintenance client. + const longGlobalDb = onMaintenanceClient(appsGlobalDb); + const infoCol = longGlobalDb.collection(stagingCol); await infoCol.createIndexes([ { key: { name: 1 }, name: 'query for getting zelapp based on zelapp specs name' }, { key: { owner: 1 }, name: 'query for getting zelapp based on zelapp specs owner' }, @@ -943,7 +992,7 @@ async function reindexGlobalAppsInformation( { key: { height: 1 }, name: 'query for getting zelapp based on last height update' }, { key: { hash: 1 }, name: 'query for getting zelapp based on last hash' }, ]); - await ensureNewestAppMessageIndex(appsGlobalDb.collection(globalAppsMessagesCol)); + await ensureNewestAppMessageIndex(longGlobalDb.collection(globalAppsMessagesCol)); const pipeline = [ { $sort: NEWEST_APP_MESSAGE_SORT }, @@ -980,7 +1029,7 @@ async function reindexGlobalAppsInformation( ]; const resultCursor = await aggregateInDatabase( - appsGlobalDb, + longGlobalDb, globalAppsMessagesCol, pipeline, { returnArray: false }, @@ -1126,9 +1175,7 @@ async function main(command) { await repairNanInAppsMessagesDb({ force: true }); } - const client = databaseConnection(); - - await client.close(); + await closeDbConnection(); } if (require.main === module) { @@ -1152,6 +1199,9 @@ if (require.main === module) { } module.exports = { + maintenanceConnection, + onMaintenanceClient, + MAINTENANCE_SOCKET_TIMEOUT_MS, aggregateInDatabase, isDatabaseError, appMessagesThatCount, diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index ac265d5d2c..5ad9aeab71 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -757,8 +757,9 @@ async function restoreDatabaseToBlockheightState(height) { if (!height) { throw new Error('No blockheight for restoring provided'); } + // Whole-collection deletes and updates: they run on the maintenance client. const dbopen = dbHelper.databaseConnection(); - const database = dbopen.db(config.database.daemon.database); + const database = dbHelper.onMaintenanceClient(dbopen.db(config.database.daemon.database)); const query = { height: { $gt: height } }; const queryForAddresses = {}; // we need to remove those transactions in transactions field that have height greater than height @@ -777,7 +778,7 @@ async function restoreDatabaseToBlockheightState(height) { // restore appsHashes collection await dbHelper.removeDocumentsFromCollection(database, appsHashesCollection, query); log.info('Rescanning Blockchain Parameters!'); - const databaseUpdates = dbopen.db(config.database.chainparams.database); + const databaseUpdates = dbHelper.onMaintenanceClient(dbopen.db(config.database.chainparams.database)); await dbHelper.removeDocumentsFromCollection(databaseUpdates, chainParamsMessagesCollection, query); log.info('Rescan completed'); return true; @@ -1236,6 +1237,8 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore) { if (scannedBlockHeight === 0) { log.info('Preparing daemon collections'); + // The index builds run on the maintenance client. + const longDatabase = dbHelper.onMaintenanceClient(database); const result = await dbHelper.dropCollection(database, utxoIndexCollection).catch((error) => { if (error.message !== 'ns not found') throw error; }); @@ -1248,35 +1251,35 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore) { const resultFusion = await dbHelper.dropCollection(database, coinbaseFusionIndexCollection).catch((error) => { if (error.message !== 'ns not found') throw error; }); - const databaseUpdates = db.db(config.database.chainparams.database); + const databaseUpdates = dbHelper.onMaintenanceClient(db.db(config.database.chainparams.database)); const resultChainParams = await dbHelper.dropCollection(databaseUpdates, chainParamsMessagesCollection).catch((error) => { if (error.message !== 'ns not found') throw error; }); log.info(result, resultB, resultD, resultFusion, resultChainParams); - await database.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting utxo', unique: true }); - await database.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting utxo for zelnode tx', unique: true }); - await database.collection(utxoIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses utxo' }); - await database.collection(utxoIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey utxo' }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting coinbase fusion utxo', unique: true }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting coinbase fusion utxo for zelnode tx', unique: true }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses coinbase fusion utxo' }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey coinbase fusion utxo' }); - await database.collection(addressTransactionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses transactions' }); - await database.collection(addressTransactionIndexCollection).createIndex({ address: 1, count: 1 }, { name: 'query for addresses transactions with count' }); - await database.collection(appsHashesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid' }); - await database.collection(appsHashesCollection).createIndex({ height: 1 }, { name: 'query for getting height' }); - await database.collection(appsHashesCollection).createIndex({ hash: 1 }, { name: 'query for getting app hash', unique: true }).catch((error) => { + await longDatabase.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting utxo', unique: true }); + await longDatabase.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting utxo for zelnode tx', unique: true }); + await longDatabase.collection(utxoIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses utxo' }); + await longDatabase.collection(utxoIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey utxo' }); + await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting coinbase fusion utxo', unique: true }); + await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting coinbase fusion utxo for zelnode tx', unique: true }); + await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses coinbase fusion utxo' }); + await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey coinbase fusion utxo' }); + await longDatabase.collection(addressTransactionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses transactions' }); + await longDatabase.collection(addressTransactionIndexCollection).createIndex({ address: 1, count: 1 }, { name: 'query for addresses transactions with count' }); + await longDatabase.collection(appsHashesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid' }); + await longDatabase.collection(appsHashesCollection).createIndex({ height: 1 }, { name: 'query for getting height' }); + await longDatabase.collection(appsHashesCollection).createIndex({ hash: 1 }, { name: 'query for getting app hash', unique: true }).catch((error) => { log.error('Expected throw on index creation as of new uniquness. Do not remove this check until all nodes have rebuild apps data'); log.error(error); }); - await database.collection(appsHashesCollection).createIndex({ message: 1 }, { name: 'query for getting app hashes depending if we have message' }); + await longDatabase.collection(appsHashesCollection).createIndex({ message: 1 }, { name: 'query for getting app hashes depending if we have message' }); await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid of some chain parameters update message' }); await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ height: 1 }, { name: 'query for getting height of some chain parameters update message' }); await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ message: 1 }, { name: 'query for getting message of some chain parameters update message' }); await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ version: 1 }, { name: 'query for getting version of some chain parameters update message' }); - const databaseGlobal = db.db(config.database.appsglobal.database); + const databaseGlobal = dbHelper.onMaintenanceClient(db.db(config.database.appsglobal.database)); log.info('Preparing apps collections'); await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ hash: 1 }, { name: 'query for getting zelapp message based on hash', unique: true }); await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ txid: 1 }, { name: 'query for getting zelapp message based on txid' }); @@ -1291,16 +1294,16 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore) { await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ repotag: 1 }, { name: 'query for getting zelapp based on image' }); await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ height: 1 }, { name: 'query for getting zelapp based on last height update' }); await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ hash: 1 }, { name: 'query for getting zelapp based on last hash' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp location based on zelapp specs name' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ hash: 1 }, { name: 'query for getting zelapp location based on zelapp hash' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ ip: 1 }, { name: 'query for getting zelapp location based on ip' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting app based on ip and name' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1, broadcastedAt: 1 }, { name: 'query for getting app to ensure we possess a message' }); - await database.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp install location based on zelapp specs name' }); - await database.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting flux app install location based on specs name and node ip' }); - await database.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1 }, { name: 'query for getting flux app install errors location based on specs name' }); - await database.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash' }); - await database.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1, ip: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash and node ip' }); + await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp location based on zelapp specs name' }); + await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ hash: 1 }, { name: 'query for getting zelapp location based on zelapp hash' }); + await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ ip: 1 }, { name: 'query for getting zelapp location based on ip' }); + await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting app based on ip and name' }); + await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1, broadcastedAt: 1 }, { name: 'query for getting app to ensure we possess a message' }); + await longDatabase.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp install location based on zelapp specs name' }); + await longDatabase.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting flux app install location based on specs name and node ip' }); + await longDatabase.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1 }, { name: 'query for getting flux app install errors location based on specs name' }); + await longDatabase.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash' }); + await longDatabase.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1, ip: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash and node ip' }); log.info('Preparation done'); } diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index a93991498a..ec84c954b2 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -303,7 +303,8 @@ async function untilSucceeds(what, step) { */ async function prepareDatabaseSchema() { log.info('Preparing local database...'); - const db = dbHelper.databaseConnection(); + // Index builds: they run on the maintenance client. + const db = dbHelper.maintenanceConnection() || dbHelper.databaseConnection(); const database = db.db(config.database.local.database); // Logins survive a restart: every request re-verifies its signature and // re-checks its privilege against the node's current config, and each diff --git a/tests/unit/dbHelper.test.js b/tests/unit/dbHelper.test.js index 7ec8aa7a5b..6e6959e30c 100644 --- a/tests/unit/dbHelper.test.js +++ b/tests/unit/dbHelper.test.js @@ -1,6 +1,7 @@ const { expect } = require('chai'); const config = require('config'); -const { ObjectId } = require('mongodb'); +const { ObjectId, MongoClient } = require('mongodb'); +const sinon = require('sinon'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const { requireMongo } = require('./dbTestHelper'); @@ -778,6 +779,48 @@ describe('dbHelper tests', () => { }); }); + describe('the maintenance client', () => { + afterEach(async () => { + sinon.restore(); + await dbHelper.closeDbConnection(); + }); + + const client = () => ({ close: sinon.stub().resolves(), db: sinon.stub().callsFake((name) => ({ databaseName: name, client: 'maintenance' })) }); + + it('is connected beside the main client, with a fifteen-minute budget for one round trip', async () => { + await dbHelper.closeDbConnection(); + const main = client(); + const maintenance = client(); + const connect = sinon.stub(MongoClient, 'connect'); + connect.onFirstCall().resolves(main); + connect.onSecondCall().resolves(maintenance); + + await dbHelper.initiateDB(); + + expect(connect.firstCall.args[1].maxPoolSize).to.equal(100); + expect(connect.secondCall.args[1]).to.include({ socketTimeoutMS: 15 * 60 * 1000, waitQueueTimeoutMS: 15 * 60 * 1000 }); + expect(dbHelper.databaseConnection()).to.equal(main); + expect(dbHelper.maintenanceConnection()).to.equal(maintenance); + expect(dbHelper.onMaintenanceClient({ databaseName: 'globalzelappstest' })).to.deep.equal({ databaseName: 'globalzelappstest', client: 'maintenance' }); + }); + + it('is connected with the main client or not at all', async () => { + await dbHelper.closeDbConnection(); + const main = client(); + const connect = sinon.stub(MongoClient, 'connect'); + connect.onFirstCall().resolves(main); + connect.onSecondCall().rejects(new Error('connection refused')); + + const error = await dbHelper.initiateDB().catch((caught) => caught); + + expect(error.message).to.equal('connection refused'); + + sinon.assert.calledOnce(main.close); + expect(dbHelper.databaseConnection()).to.equal(null); + expect(dbHelper.maintenanceConnection()).to.equal(null); + }); + }); + describe('repairNanInAppsMessagesDb', () => { const { appsglobal, daemon, local } = config.database; let messages; diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 0fba8638bb..55696a181c 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -1212,6 +1212,17 @@ describe('explorerService tests', () => { sinon.assert.calledWithMatch(updateInDatabaseStub, sinon.match.object, 'addresstransactionindex', {}, { $pull: { transactions: { height: sinon.match.object } } }); }); + it('runs its deletes and updates on the maintenance client', async () => { + removeDocumentsFromCollectionStub.returns(true); + updateInDatabaseStub.returns(true); + + await explorerService.restoreDatabaseToBlockheightState(100000); + + const clients = [...removeDocumentsFromCollectionStub.getCalls(), ...updateInDatabaseStub.getCalls()].map((call) => call.args[0].client); + expect(clients).to.have.length.above(0); + clients.forEach((client) => expect(client).to.equal(dbHelper.maintenanceConnection())); + }); + it('should leave the app messages and the registry alone', async () => { removeDocumentsFromCollectionStub.returns(true); updateInDatabaseStub.returns(true); diff --git a/tests/unit/registryRebuild.test.js b/tests/unit/registryRebuild.test.js index a256cc83af..a649592f3e 100644 --- a/tests/unit/registryRebuild.test.js +++ b/tests/unit/registryRebuild.test.js @@ -11,6 +11,7 @@ const fs = require('fs'); const path = require('path'); const { expect } = require('chai'); +const sinon = require('sinon'); const { MongoNetworkTimeoutError } = require('mongodb'); const espree = require('espree'); @@ -277,6 +278,28 @@ describe('registry rebuild: who holds a name', () => { expect(await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb)).to.equal(true); }); + it('runs its index builds and its $group on the maintenance client', async () => { + await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); + const onMaintenance = sinon.spy(dbHelper.maintenanceConnection(), 'db'); + try { + await rebuild(); + } finally { + onMaintenance.restore(); + } + sinon.assert.calledWith(onMaintenance, globalDb.databaseName); + }); + + it('runs the reindex check\'s index build and $group on the maintenance client', async () => { + await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); + const onMaintenance = sinon.spy(dbHelper.maintenanceConnection(), 'db'); + try { + await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb); + } finally { + onMaintenance.restore(); + } + sinon.assert.calledWith(onMaintenance, globalDb.databaseName); + }); + it('rejects when the registry cannot be read, rather than reading it as consistent', async () => { await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); await rebuild(); diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index 7061a4918c..481c664c2e 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -175,10 +175,11 @@ describe('serviceManager prepareDatabaseSchema', () => { afterEach(() => sinon.restore()); it('rejects when the login rows cannot be cleared of signatures, so boot prepares the schema again', async () => { - sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({ collection: () => ({}) }) }); - sinon.stub(dbHelper, 'updateInDatabase').rejects(new Error('connection timed out')); + sinon.stub(dbHelper, 'maintenanceConnection').returns({ db: (name) => ({ name, client: 'maintenance' }) }); + const unset = sinon.stub(dbHelper, 'updateInDatabase').rejects(new Error('connection timed out')); await expect(prepareDatabaseSchema()).to.be.rejectedWith('connection timed out'); + expect(unset.firstCall.args[0].client, 'the schema runs on the maintenance client').to.equal('maintenance'); }); }); From 8cbd980f0e3467290a8f1519f328883e0d8d1c8b Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:06:16 +0100 Subject: [PATCH 078/369] feat(db): a node below MongoDB 6.0.27 is held out of service config.minimumMongoDbAllowedVersion is 6.0.27. Boot reads the MongoDB version as soon as the database answers, and a version below the floor sets the sticky DOS owner MONGODB_FLOOR, naming the version found and the version required. A version that could not be read, or is not SemVer, gives no verdict. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 4 +++ ZelBack/src/services/fluxNetworkHelper.js | 38 ++++++++++++++++++++ ZelBack/src/services/serviceManager.js | 3 ++ tests/unit/fluxNetworkHelper.test.js | 42 +++++++++++++++++++++++ tests/unit/globalconfig/default.js | 1 + 5 files changed, 88 insertions(+) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index f7b06e863e..9dfe5652d4 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -136,6 +136,10 @@ module.exports = { // Docker's networking rules and its patch releases fixed them; 28.1.1 is the // oldest release the fleet runs in numbers. minimumDockerAllowedVersion: '28.1.1', + // The MongoDB a node runs: the oldest release FluxOS supports. From 6.0 a + // $group or $sort past the memory limit spills to disk rather than failing, + // which the registry rebuild and the duplicate merges at boot rely on. + minimumMongoDbAllowedVersion: '6.0.27', fluxTeamFluxID: '1hjy4bCYBJr4mny4zCE85J94RXa8W6q37', // A list, so support can be granted to (or revoked from) an identity without // touching every privilege check. A bare string is still read as a one entry diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 753c1847d1..e3355ac394 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -97,6 +97,7 @@ const StickyDosOwner = Object.freeze({ PEER_SET_STABILITY: 'peerSetStability', NODEJS_FLOOR: 'nodejsFloor', DOCKER_FLOOR: 'dockerFloor', + MONGODB_FLOOR: 'mongodbFloor', }); // Declares the node unfit for the apps it already runs, and is not moved by the @@ -1304,6 +1305,42 @@ function checkDockerVersionAllowed(dockerVersion) { return false; } +/** + * Whether the MongoDB this node runs meets the network minimum: the oldest + * release FluxOS supports. + * + * Asked once, when MongoDB first answers at startup. Upgrading MongoDB + * restarts mongod, not FluxOS, so the verdict stands until FluxOS next starts, + * and the message says so. A version that could not be read, or is not a + * SemVer version, gives no verdict. A pre-release is below its release. + * @param {?string} mongoDbVersion The version MongoDB reports, or null when it + * could not be read. + * @returns {boolean} True unless the version is known and below the floor. + */ +function checkMongoDbVersionAllowed(mongoDbVersion) { + const minimumVersion = config.minimumMongoDbAllowedVersion; + if (!minimumVersion) { + log.error('checkMongoDbVersionAllowed - no minimum MongoDB version configured, skipping the check'); + return true; + } + if (!mongoDbVersion) { + log.error('checkMongoDbVersionAllowed - MongoDB did not report its version, no verdict'); + return true; + } + if (!serviceHelper.parseSemver(mongoDbVersion)) { + log.error(`checkMongoDbVersionAllowed - MongoDB reported ${mongoDbVersion}, which is not a SemVer version, no verdict`); + return true; + } + if (serviceHelper.semverAtLeast(mongoDbVersion, minimumVersion)) { + return true; + } + setStickyDos( + StickyDosOwner.MONGODB_FLOOR, + `MongoDB Version Error. Current lower version allowed is v${minimumVersion} found v${mongoDbVersion}. Upgrade MongoDB and restart FluxOS`, + ); + return false; +} + /** * To check if Flux benchmark version is allowed. * @returns {boolean} True if version is verified as allowed. Otherwise false. @@ -3526,6 +3563,7 @@ module.exports = { checkFluxbenchVersionAllowed, checkNodeJsVersionAllowed, checkDockerVersionAllowed, + checkMongoDbVersionAllowed, checkMyFluxAvailability, adjustExternalIp, handleOwnAddress, diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index ec84c954b2..a0356f16a8 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -454,6 +454,9 @@ async function startFluxFunctions() { fluxNetworkHelper.checkNodeJsVersionAllowed(); // Hard dependencies — nothing starts until these are confirmed. await dbHelper.waitForMongo(); + // As soon as MongoDB answers: a below-floor node stays up holding a sticky + // DOS, so /flux/info names the version found and the version required. + fluxNetworkHelper.checkMongoDbVersionAllowed(await dbHelper.getMongoDbVersion()); await dockerService.waitForDocker(); // As soon as Docker answers, before the container firewall is applied: a // below-floor node stays up holding a sticky DOS, so /flux/info names the diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 3ef3f02239..40410b516f 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -1081,6 +1081,48 @@ describe('fluxNetworkHelper tests', () => { }); }); + describe('checkMongoDbVersionAllowed tests', () => { + // minimumMongoDbAllowedVersion = '6.0.27' + const { MONGODB_FLOOR } = fluxNetworkHelper.StickyDosOwner; + + afterEach(() => { + fluxNetworkHelper.clearStickyDos(MONGODB_FLOOR); + fluxNetworkHelper.setDosStateValue(0); + fluxNetworkHelper.setDosMessage(null); + }); + + it('allows the MongoDB the fleet runs, and the floor itself', () => { + ['8.0.20', '7.0.31', '6.0.27'].forEach((version) => { + expect(fluxNetworkHelper.checkMongoDbVersionAllowed(version), version).to.equal(true); + }); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + }); + + it('takes a node below the floor out of service, and says which version it found and what to do', () => { + expect(fluxNetworkHelper.checkMongoDbVersionAllowed('6.0.26')).to.equal(false); + const reported = fluxNetworkHelper.getDOSState().data; + expect(reported.dosMessage).to.include('6.0.27'); + expect(reported.dosMessage).to.include('6.0.26'); + expect(reported.dosMessage).to.include('restart FluxOS'); + expect(reported.dosState).to.equal(100); + }); + + it('refuses every release below the floor the fleet still runs, and a pre-release of the floor', () => { + ['4.4.29', '4.4.18', '6.0.26', '6.0.27-rc0'].forEach((version) => { + fluxNetworkHelper.clearStickyDos(MONGODB_FLOOR); + expect(fluxNetworkHelper.checkMongoDbVersionAllowed(version), version).to.equal(false); + }); + }); + + it('gives no verdict when MongoDB did not report its version, or reports one that is not SemVer', () => { + [null, undefined, '', '6.0', 'dev'].forEach((version) => { + expect(fluxNetworkHelper.checkMongoDbVersionAllowed(version), String(version)).to.equal(true); + }); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + expect(fluxNetworkHelper.getDOSState().data.dosState).to.equal(0); + }); + }); + describe('checkDockerVersionAllowed tests', () => { // minimumDockerAllowedVersion = '28.1.1' const { DOCKER_FLOOR, RESIDENTIAL_DOS } = fluxNetworkHelper.StickyDosOwner; diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 4f6baa1549..044e4f49ff 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -109,6 +109,7 @@ module.exports = { minimumNodeJsAllowedVersion: '20.8.0', minimumSyncthingAllowedVersion: '1.27.6', minimumDockerAllowedVersion: '28.1.1', + minimumMongoDbAllowedVersion: '6.0.27', fluxTeamFluxID: '1NH9BP155Rp3HSf5ef6NpUbE8JcyLRruAM', fluxSupportTeamFluxID: ['16iJqiVbHptCx87q6XQwNpKdgEZnFtKcyP'], deterministicNodesStart: 558000, From fe10539f579b38315981d1ec1761804f6ed979a2 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:06:47 +0100 Subject: [PATCH 079/369] fix(db): a database call that gets no answer fails after a minute The main MongoDB client sets socketTimeoutMS and waitQueueTimeoutMS to 60 s, so a round trip, or the wait for a pooled connection, ends in an error instead of waiting for ever. A read is retried once by the driver and fails after two minutes; a write is not retried. The maintenance client keeps its 15 minute bounds. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 10 ++++++++++ tests/unit/dbHelper.test.js | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index 04e804f019..94146ffa61 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -19,6 +19,11 @@ const mongoUrl = `mongodb://${config.database.url}:${config.database.port}/`; */ let openDBConnection = null; +// The main client's bound on one round trip and on the wait for a pooled +// connection: over four times the slowest healthy round trip measured on the +// fleet. +const MAIN_SOCKET_TIMEOUT_MS = 60 * 1000; + // The client for the named long operations: index builds, the explorer // restore's deletes and the registry rebuild. One of its round trips may take // up to MAINTENANCE_SOCKET_TIMEOUT_MS; every other call uses the main client. @@ -92,6 +97,11 @@ async function connectMongoDb(url, settings = {}) { const connectUrl = url || mongoUrl; const mongoSettings = { maxPoolSize: 100, + // One round trip, and the wait for a pooled connection, end in an error + // after a minute instead of waiting for ever. A read is retried once, so it + // fails after two; a write is not, and its outcome is then unknown. + socketTimeoutMS: MAIN_SOCKET_TIMEOUT_MS, + waitQueueTimeoutMS: MAIN_SOCKET_TIMEOUT_MS, ...settings, }; const client = await MongoClient.connect(connectUrl, mongoSettings); diff --git a/tests/unit/dbHelper.test.js b/tests/unit/dbHelper.test.js index 6e6959e30c..be7ae4db5f 100644 --- a/tests/unit/dbHelper.test.js +++ b/tests/unit/dbHelper.test.js @@ -787,7 +787,7 @@ describe('dbHelper tests', () => { const client = () => ({ close: sinon.stub().resolves(), db: sinon.stub().callsFake((name) => ({ databaseName: name, client: 'maintenance' })) }); - it('is connected beside the main client, with a fifteen-minute budget for one round trip', async () => { + it('is connected beside the main client, with fifteen minutes for one round trip where the main client has one', async () => { await dbHelper.closeDbConnection(); const main = client(); const maintenance = client(); @@ -797,7 +797,7 @@ describe('dbHelper tests', () => { await dbHelper.initiateDB(); - expect(connect.firstCall.args[1].maxPoolSize).to.equal(100); + expect(connect.firstCall.args[1]).to.include({ maxPoolSize: 100, socketTimeoutMS: 60 * 1000, waitQueueTimeoutMS: 60 * 1000 }); expect(connect.secondCall.args[1]).to.include({ socketTimeoutMS: 15 * 60 * 1000, waitQueueTimeoutMS: 15 * 60 * 1000 }); expect(dbHelper.databaseConnection()).to.equal(main); expect(dbHelper.maintenanceConnection()).to.equal(maintenance); From b92796565533ff9cedcbbd4e5829a28f5b2a71e5 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:10:51 +0100 Subject: [PATCH 080/369] test(harness): 3301 pauses the fleet's mongod past a read's whole budget, while the fleet runs and while a node boots mongo-control pauses and resumes the shared mongod. With it paused for 150 s the fleet keeps every FluxOS process, removes and announces no app, keeps an operator-stopped component stopped, and announces its apps again once mongod answers. Paused while a node boots, the boot holds and then runs once. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/mongo-control.js | 20 ++ ...that-stops-answering-takes-nothing-down.js | 178 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 test-infra/runner/framework/mongo-control.js create mode 100644 test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js new file mode 100644 index 0000000000..a8bb21cb70 --- /dev/null +++ b/test-infra/runner/framework/mongo-control.js @@ -0,0 +1,20 @@ +import { getContainerRuntimeClient } from 'testcontainers'; + +// The fleet's one mongod, held still and let go. A paused container keeps its +// sockets open and answers nothing, which is what a mongod whose thread is stuck +// in the kernel looks like from FluxOS: every call waits, none fails. The harness's +// own database client (dbClient) waits with it, so a suite reads nothing through it +// while mongod is paused. + +async function mongoContainer(env) { + const client = await getContainerRuntimeClient(); + return client.container.dockerode.getContainer(env.containers.mongo.getId()); +} + +export async function pauseMongo(env) { + await (await mongoContainer(env)).pause(); +} + +export async function resumeMongo(env) { + await (await mongoContainer(env)).unpause(); +} diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js new file mode 100644 index 0000000000..1dc4595a12 --- /dev/null +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -0,0 +1,178 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { + execInContainer, getAppContainerStatus, crashFluxos, releaseFluxos, +} from '../framework/container.js'; +import { + waitFor, waitForUp, waitForDown, waitForOperatorIntent, waitForBootSettled, assertNoEvent, +} from '../framework/wait.js'; +import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; +import { pauseMongo, resumeMongo } from '../framework/mongo-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A mongod that stops answering, and comes back. The fleet shares one mongod, and a +// paused container is what a mongod whose thread is stuck in the kernel looks like +// from FluxOS: every call waits and none fails. FluxOS bounds a call at 60 s (a read +// is retried once, so 120 s), and every caller of a call that fails leaves what it +// cannot read as it is. Paused for longer than a read's whole budget, the fleet: +// +// - keeps every FluxOS process it had, so nothing exits on a failed call; +// - removes no app and announces no removal; +// - keeps an operator-stopped component stopped; +// - announces its apps again once mongod answers. +// +// The second case pauses mongod while a node boots: the boot holds until mongod +// answers and runs once, rather than starting again from the top. + +const subnet = getSubnetConfig(); +// Over a read's whole budget: 60 s, and the driver's one retry. +const PAUSE_MS = 150 * 1000; +// What a call that got no answer leaves in a node's log. One of them shows the +// pause reached FluxOS, so the assertions below are about a stall, not a quiet minute. +const DATABASE_GAVE_NO_ANSWER = /MongoNetworkTimeoutError|MongoWaitQueueTimeoutError|MongoServerSelectionError|timed out/; + +async function fluxosPid(client) { + return Number((await execInContainer(client.container, 'cat /tmp/fluxos.pid')).stdout.trim()); +} + +describe('a database that stops answering takes nothing down', function () { + let env; + dumpLogsOnFailure(() => env); + const runningName = `e2edbrun${Date.now()}`; + const stoppedName = `e2edbstop${Date.now()}`; + let runningIdx; + let stoppedIdx; + let paused = false; + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ hookCtx: this, nodes: 10, tickerAutostart: false }); + await bootAndPeer(env); + ({ index: runningIdx } = await seedSimpleApp(env, runningName)); + ({ index: stoppedIdx } = await seedSimpleApp(env, stoppedName)); + await waitForUp(env.clients[runningIdx], runningName, 'the running app is up'); + await waitForUp(env.clients[stoppedIdx], stoppedName, 'the app to stop is up'); + + const client = env.clients[stoppedIdx]; + const auth = await authenticate(client.url, appOwnerKey()); + const stopRes = await client.getAuthed(`/apps/appstop/${stoppedName}`, auth.zelidauth); + expect(stopRes.status, `appstop refused: ${JSON.stringify(stopRes.data)}`).to.equal('success'); + await waitForOperatorIntent(client, `${stoppedName}_${stoppedName}`, true); + await waitForDown(client, stoppedName, 'stopped by its owner'); + }); + + after(async function () { + this.timeout(60000); + if (paused) await resumeMongo(env).catch(() => {}); + await env?.teardown(); + }); + + describe('while the fleet runs', function () { + let pidsBefore; + let anchors; + + before(async function () { + this.timeout(PAUSE_MS + 120000); + pidsBefore = await Promise.all(env.clients.map(fluxosPid)); + anchors = env.clients.map((client) => client.getLastEventId()); + await pauseMongo(env); + paused = true; + try { + await new Promise((resolve) => { setTimeout(resolve, PAUSE_MS); }); + } finally { + await resumeMongo(env); + paused = false; + } + }); + + it('reached FluxOS: a node logged a call the database did not answer', function () { + const stalled = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); + expect(stalled, 'no node logged a database call that timed out, so nothing below was tested').to.not.be.empty; + }); + + it('keeps every FluxOS process it had', async function () { + const pidsAfter = await Promise.all(env.clients.map(fluxosPid)); + expect(pidsAfter, 'a node\'s FluxOS exited during the pause').to.deep.equal(pidsBefore); + }); + + it('removes no app and announces no removal', async function () { + this.timeout(30000); + await Promise.all(env.clients.map(async (client, i) => { + await assertNoEvent(client, 'app:removed', () => true, 1, { afterId: anchors[i] }); + await assertNoEvent(client, 'network:appremoved', () => true, 1, { afterId: anchors[i] }); + })); + const running = await getAppContainerStatus(env.clients[runningIdx].container, runningName); + expect(running && running.status.startsWith('Up'), 'the running app is still up').to.equal(true); + }); + + it('keeps the operator-stopped component stopped', async function () { + this.timeout(30000); + const client = env.clients[stoppedIdx]; + await assertNoEvent(client, 'reconciler:actuated', (d) => d.identifier === `${stoppedName}_${stoppedName}` && d.action === 'started', 1, { afterId: anchors[stoppedIdx] }); + const stopped = await getAppContainerStatus(client.container, stoppedName, { all: true }); + expect(stopped && stopped.status.startsWith('Up'), 'the stopped component was started').to.not.equal(true); + }); + + it('announces its apps again once the database answers', async function () { + this.timeout(240000); + const holderIp = subnet.nodeIp(runningIdx + 1); + const peer = env.clients[(runningIdx + 1) % env.clients.length]; + const afterId = peer.getLastEventId(); + await peer.waitForEvent( + 'network:apprunning', + (d) => d.ip?.startsWith(holderIp) && d.apps?.some((a) => a.name === runningName), + 200000, + { afterId }, + ); + const installed = await env.clients[runningIdx].get('/apps/installedapps'); + expect(installed.status, 'the node reads its installed apps again').to.equal('success'); + expect(installed.data.map((app) => app.name)).to.include(runningName); + }); + }); + + describe('while a node boots', function () { + const idx = 0; + // One line from the stretch of boot before the schema, and one from after it: a + // boot that started again from the top would log either of them twice. + const BEFORE_SCHEMA = /System service initiated/; + const AFTER_SCHEMA = /Flux checks operational/; + let counts; + let bootAnchor; + + before(async function () { + this.timeout(PAUSE_MS + 300000); + const { container } = env.clients[idx]; + // Event ids are seeded from the clock and keep rising across a restart. + bootAnchor = env.clients[idx].getLastEventId(); + await crashFluxos(container, { hold: true }); + counts = { before: env.nodeLogCount(idx, BEFORE_SCHEMA), after: env.nodeLogCount(idx, AFTER_SCHEMA) }; + await pauseMongo(env); + paused = true; + try { + await releaseFluxos(container); + await new Promise((resolve) => { setTimeout(resolve, PAUSE_MS); }); + } finally { + await resumeMongo(env); + paused = false; + } + }); + + it('held the boot while the database gave no answer', function () { + expect(env.nodeHasLog(idx, /Waiting for MongoDB/), 'the boot never waited on the database, so nothing below was tested').to.equal(true); + }); + + it('finishes the boot once, after the database answers', async function () { + this.timeout(240000); + await waitForBootSettled(env.clients[idx], 200000, { afterId: bootAnchor }); + await waitFor(() => env.nodeLogCount(idx, AFTER_SCHEMA) > counts.after, { + timeout: 60000, interval: 2000, label: 'the boot reaches the start of the flux checks', + }); + expect(env.nodeLogCount(idx, BEFORE_SCHEMA) - counts.before, 'the stretch before the schema ran').to.equal(1); + expect(env.nodeLogCount(idx, AFTER_SCHEMA) - counts.after, 'the start phase ran').to.equal(1); + }); + }); +}); From d9c299dcd51a7a97557f559bb28516bc10c172c2 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:27:13 +0100 Subject: [PATCH 081/369] perf(db): the payment position backfill does not read again a record not on the chain backfillPaymentPositions reads only the payment records that have no position and are not marked notOnChain. A marked record is decided: when its app message is paid in a later block, the explorer records that payment in its place. A node holding one no longer repeats the backfill's address index call on every boot. Co-Authored-By: Claude Opus 5.5 --- .../services/migrations/paymentPositions.js | 6 ++- tests/unit/paymentPositions.test.js | 38 ++++++++++++++++++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/migrations/paymentPositions.js b/ZelBack/src/services/migrations/paymentPositions.js index f0b6b6b409..c63fa4fd94 100644 --- a/ZelBack/src/services/migrations/paymentPositions.js +++ b/ZelBack/src/services/migrations/paymentPositions.js @@ -15,7 +15,7 @@ const BACKFILL_BATCH = 5000; * and the height the daemon's address index holds for its transaction: one call * over the payment addresses from the first height a payment is recorded at to * the chain tip, so a transaction the index does not hold is not on the chain. - * Such a record is marked `notOnChain`, and is no payment. A record above the + * Such a record is marked `notOnChain`, is no payment, and is not read again. A record above the * daemon's tip is left for a later run: the daemon has not reached its block. * * The height is the chain's, not the one recorded: a record written from a block @@ -32,8 +32,10 @@ const BACKFILL_BATCH = 5000; async function backfillPaymentPositions() { const database = dbHelper.databaseConnection().db(config.database.daemon.database); const collection = config.database.daemon.collections.appsHashes; + // A record already marked notOnChain is decided: if its transaction is mined + // later, the explorer records the payment from that block. const missing = await dbHelper.findInDatabase( - database, collection, { txIndex: { $exists: false } }, { + database, collection, { txIndex: { $exists: false }, notOnChain: { $ne: true } }, { projection: { _id: 0, hash: 1, txid: 1, height: 1, value: 1, }, diff --git a/tests/unit/paymentPositions.test.js b/tests/unit/paymentPositions.test.js index 023afd6043..8db1626197 100644 --- a/tests/unit/paymentPositions.test.js +++ b/tests/unit/paymentPositions.test.js @@ -12,6 +12,7 @@ const daemonServiceUtils = require('../../ZelBack/src/services/daemonService/dae const messageVerifier = require('../../ZelBack/src/services/appMessaging/messageVerifier'); const { appPaymentAddresses, appPaymentPositions } = require('../../ZelBack/src/services/appPaymentPositions'); const { backfillPaymentPositions, BACKFILL_BATCH } = require('../../ZelBack/src/services/migrations/paymentPositions'); +const { requireMongo } = require('./dbTestHelper'); chai.use(chaiAsPromised); const { expect } = chai; @@ -85,6 +86,41 @@ describe('app payment positions', () => { }); }); + describe('backfillPaymentPositions against the database', () => { + before(requireMongo); + + let hashes; + + beforeEach(async () => { + await dbHelper.initiateDB(); + hashes = dbHelper.databaseConnection().db(config.database.daemon.database).collection(config.database.daemon.collections.appsHashes); + await hashes.deleteMany({}); + sinon.stub(messageVerifier, 'moveStoredMessagesToPayments').resolves([]); + sinon.stub(messageVerifier, 'storeGoverningAppSpecifications').resolves(); + executeCall.withArgs('getBlockCount').resolves({ status: 'success', data: 1000 }); + executeCall.withArgs('getaddressdeltas').resolves({ status: 'success', data: [] }); + }); + + afterEach(async () => { + await hashes.deleteMany({}); + }); + + it('asks the daemon nothing for a record it already found not on the chain', async () => { + await hashes.insertOne({ + hash: 'orphaned', txid: 'orphaned', height: 500, value: 1, + }); + + expect((await backfillPaymentPositions()).notOnChain).to.equal(1); + expect((await hashes.findOne({ hash: 'orphaned' })).notOnChain).to.equal(true); + executeCall.resetHistory(); + + expect(await backfillPaymentPositions()).to.deep.equal({ + positioned: 0, heightsCorrected: 0, notOnChain: 0, aboveTip: 0, + }); + sinon.assert.notCalled(executeCall); + }); + }); + describe('backfillPaymentPositions', () => { let findInDatabase; let bulkWriteInDatabase; @@ -117,7 +153,7 @@ describe('app payment positions', () => { findInDatabase.resolves([]); await backfillPaymentPositions(); expect(findInDatabase.firstCall.args[1]).to.equal(config.database.daemon.collections.appsHashes); - expect(findInDatabase.firstCall.args[2]).to.deep.equal({ txIndex: { $exists: false } }); + expect(findInDatabase.firstCall.args[2]).to.deep.equal({ txIndex: { $exists: false }, notOnChain: { $ne: true } }); }); it('positions a record, at the height the chain holds its transaction', async () => { From bc781c1e7d65fc7d6be6d67ca05bc85c2447497d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:40:18 +0100 Subject: [PATCH 082/369] perf(db): boot no longer drops the retired broadcastedAt and cachedAt indexes The TTL on these collections is expireAt, and every node at or above the 8.18.0 floor has already dropped the broadcastedAt and cachedAt TTL indexes it replaced. Boot now asserts the indexes the collections need and drops none, so the plain broadcastedAt index on the signed broadcast collections is no longer dropped and rebuilt on every boot. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/serviceManager.js | 26 +------------------------- tests/unit/serviceManager.test.js | 25 +------------------------ 2 files changed, 2 insertions(+), 49 deletions(-) diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index a0356f16a8..46e83258ad 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -216,23 +216,6 @@ async function ensureIndex(collection, spec, options = {}, recover = null) { } } -/** - * Drops the index `name` from `collection` when it is there. Rejects on any - * failure other than the index or its collection not existing, so boot - * prepares the schema again. - * @param {object} collection - a mongo collection handle - * @param {string} name - the index name - * @returns {Promise} - */ -async function dropIndexIfPresent(collection, name) { - try { - await collection.dropIndex(name); - } catch (error) { - if (error?.codeName === 'IndexNotFound' || error?.codeName === 'NamespaceNotFound') return; - throw error; - } -} - /** * Assert every index one collection needs, in a single command. * @@ -380,8 +363,7 @@ async function prepareDatabaseSchema() { { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, ]); await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); - // TTL is driven by expireAt (set per-document by store functions). Migrate from old broadcastedAt-based TTL. - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), 'broadcastedAt_1'); + // TTL is driven by expireAt (set per-document by store functions). await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, @@ -399,22 +381,18 @@ async function prepareDatabaseSchema() { { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, ]); log.info('App state events collection prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { broadcastedAt: 1 } }, { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, ]); log.info('Signed appinstalling broadcasts collection prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, ]); log.info('Flux Apps installing locations prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'cachedAt_1'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, @@ -422,7 +400,6 @@ async function prepareDatabaseSchema() { { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, ]); log.info('App installing errors locations prepared'); - await dropIndexIfPresent(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), 'broadcastedAt_1'); await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { broadcastedAt: 1 } }, @@ -1012,7 +989,6 @@ module.exports = { startFluxFunctions, ensureIndex, ensureIndexes, - dropIndexIfPresent, untilSucceeds, prepareDatabaseSchema, dedupeByKey, diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index 481c664c2e..860ea1fb7c 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -13,7 +13,7 @@ chai.use(chaiAsPromised); const { expect } = chai; const { - ensureIndex, ensureIndexes, dropIndexIfPresent, untilSucceeds, prepareDatabaseSchema, dedupeByKey, dosState, + ensureIndex, ensureIndexes, untilSucceeds, prepareDatabaseSchema, dedupeByKey, dosState, } = require('../../ZelBack/src/services/serviceManager'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); @@ -130,29 +130,6 @@ describe('serviceManager ensureIndex', () => { }); }); -describe('serviceManager dropIndexIfPresent', () => { - const serverError = (code, codeName) => Object.assign(new Error(codeName), { code, codeName }); - - it('drops the index', async () => { - const dropIndex = sinon.stub().resolves(); - - await dropIndexIfPresent({ dropIndex }, 'broadcastedAt_1'); - - sinon.assert.calledOnceWithExactly(dropIndex, 'broadcastedAt_1'); - }); - - it('settles when the index or its collection is not there', async () => { - await dropIndexIfPresent({ dropIndex: sinon.stub().rejects(serverError(27, 'IndexNotFound')) }, 'broadcastedAt_1'); - await dropIndexIfPresent({ dropIndex: sinon.stub().rejects(serverError(26, 'NamespaceNotFound')) }, 'broadcastedAt_1'); - }); - - it('rejects when the database does not answer, so boot prepares the schema again', async () => { - const timedOut = Object.assign(new Error('connection timed out'), { name: 'MongoNetworkTimeoutError' }); - - await expect(dropIndexIfPresent({ dropIndex: sinon.stub().rejects(timedOut) }, 'broadcastedAt_1')).to.be.rejectedWith('timed out'); - }); -}); - describe('serviceManager untilSucceeds', () => { afterEach(() => sinon.restore()); From 54acdf98af4839081cb8ddcb1f110a0dc7beefd2 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:41:44 +0100 Subject: [PATCH 083/369] refactor(db): the index builders live in dbHelper ensureIndex, ensureIndexes and dedupeByKey move from serviceManager to dbHelper, with their tests, so a service that owns collections can assert their indexes without depending on boot. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 146 +++++++++++++ ZelBack/src/services/serviceManager.js | 189 ++--------------- tests/unit/dbHelperIndexes.test.js | 274 +++++++++++++++++++++++++ tests/unit/serviceManager.test.js | 267 +----------------------- 4 files changed, 438 insertions(+), 438 deletions(-) create mode 100644 tests/unit/dbHelperIndexes.test.js diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index 94146ffa61..abe3a871b1 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -193,6 +193,149 @@ async function closeDbConnection() { } } +/** + * Remove rows that duplicate a would-be-unique key, keeping the newest of each. + * + * A recovery strategy for ensureIndex: when a unique build fails because the + * collection already holds rows that violate it, this makes the data conform to + * the invariant the index DECLARES - it deletes duplicates on the key the index + * says must be unique, which is enforcing a contract rather than losing data. + * Only safe where the key IS the row's identity, so it is passed in per build by + * the caller that knows the collection, never applied by default. A rollup whose + * duplicates must be summed rather than dropped (the tampering incident count) + * belongs to its owning service instead - see the note on ensureIndex. + * + * Keeps the newest per group (rows sort by _id, which is time-ordered), honours + * the index's partialFilterExpression so it only touches rows the index covers, + * and returns how many it removed. + * + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key, e.g. { hash: 1 } + * @param {object} options - the index options (read for partialFilterExpression) + * @returns {Promise} rows removed + */ +async function dedupeByKey(collection, spec, options = {}) { + const groupId = {}; + Object.keys(spec).forEach((key, i) => { groupId[`k${i}`] = `$${key}`; }); + // Held in memory deliberately, and measured rather than assumed: run against a + // live node's collection unioned with itself until every key appeared 16 times + // - 1,030,208 rows, the duplicate state this exists to repair - it finished in + // under 2s without spilling. The $sort adds nothing on top while it stays an + // index walk on _id, which it is for a spec with no partialFilterExpression; + // the first partial index to use this wants re-measuring, because the $match + // ahead of the sort is what would make the sort blocking. allowDiskUse is not + // set: it would take a mongo below 6.0, where the cap errors instead of + // spilling, and the network floor is moving past that. + // + // ids[0] rather than $max: $group does not document that it carries a + // preceding sort into an accumulator, and that non-guarantee is about results + // merged from several sources - this is one standalone mongod. Checked against + // 64,388 real duplicate groups, ids[0] was the newest in all 64,388. + const pipeline = [ + ...(options.partialFilterExpression ? [{ $match: options.partialFilterExpression }] : []), + { $sort: { _id: -1 } }, + { $group: { _id: groupId, ids: { $push: '$_id' } } }, + { $match: { 'ids.1': { $exists: true } } }, + ]; + const groups = await collection.aggregate(pipeline).toArray(); + const toRemove = groups.flatMap((group) => group.ids.slice(1)); + if (!toRemove.length) return 0; + await collection.deleteMany({ _id: { $in: toRemove } }); + return toRemove.length; +} + +/** + * Assert one index, healing the failures that are recoverable. + * + * - a pre-existing index with conflicting OPTIONS (IndexOptionsConflict / + * IndexKeySpecsConflict) is dropped by its real name and recreated; + * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` + * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the + * index rather than running degraded without it; + * - anything else rethrows. + * + * The rethrow is deliberate. Index setup runs in serviceManager's + * prepareDatabaseSchema, which boot runs again every 15 s until it succeeds: a TRANSIENT + * failure (mongo mid-election, a slow-disk blip) heals on the next pass instead + * of being skipped until the next reboot, + * and a genuinely UNRECOVERABLE database wedges loudly - which is correct, since + * a node whose DB cannot hold its schema cannot serve apps and appremove would + * not rescue it. The realistic wedge that finding motivated - a unique index + * over rows that already violate it - is repaired above, not hidden. + * + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key + * @param {object} [options] - the index options + * @param {(collection: object, spec: object, options: object) => Promise} [recover] + * run when a unique build is blocked by existing duplicate rows + */ +async function ensureIndex(collection, spec, options = {}, recover = null) { + try { + await collection.createIndex(spec, options); + } catch (err) { + const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); + if (conflict) { + const specKeys = JSON.stringify(spec); + const indexes = await collection.listIndexes().toArray(); + const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); + if (match?.name) { + log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); + await collection.dropIndex(match.name); + } + await collection.createIndex(spec, options); + return; + } + const duplicate = err && (err.code === 11000 || err.codeName === 'DuplicateKey'); + if (duplicate && recover) { + const removed = await recover(collection, spec, options); + log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); + await collection.createIndex(spec, options); + return; + } + throw err; + } +} + +/** + * Assert every index one collection needs, in a single command. + * + * mongo's index build protocol has a fixed cost per BUILD - register, start, + * scan, wait for commit quorum, commit, log - and it does not care that the + * collection is empty. A node asserting its schema one index at a time pays + * that cost 34 times over 14 collections; createIndexes pays it once per + * collection. A node boots faster for it, and the integration harness, where + * ten nodes share one mongod and every database is new, feels it ten times over + * (measured: 938 concurrent builds per ten-node fleet). + * + * The batch is the fast path, not the only one. ensureIndex heals two failures + * that need to be attributed to a single index - an options conflict it drops + * and rebuilds, and a unique build blocked by duplicate rows it repairs through + * the caller's strategy - and a batch rejection does not say which member + * failed. So any error falls back to asserting them one at a time. + * + * @param {object} collection - a mongo collection handle + * @param {Array} specs - `{ key, ...indexOptions, recover }` per index, + * where `recover` is ours and never reaches mongo + */ +async function ensureIndexes(collection, specs) { + try { + await collection.createIndexes(specs.map((spec) => { + // `recover` is a FluxOS concern; mongo is handed the index model alone + const model = { ...spec }; + delete model.recover; + return model; + })); + return; + } catch (error) { + log.warn(`ensureIndexes - batch of ${specs.length} on ${collection.collectionName} failed (${error.codeName || error.message}); asserting one at a time`); + } + // eslint-disable-next-line no-restricted-syntax + for (const { key, recover = null, ...options } of specs) { + // eslint-disable-next-line no-await-in-loop + await ensureIndex(collection, key, options, recover); + } +} + /** * Returns an array of distinct values in a given collection. * @@ -1209,6 +1352,9 @@ if (require.main === module) { } module.exports = { + ensureIndex, + ensureIndexes, + dedupeByKey, maintenanceConnection, onMaintenanceClient, MAINTENANCE_SOCKET_TIMEOUT_MS, diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 46e83258ad..3a50a0f55b 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -105,158 +105,6 @@ const dosState = { const portsNotWorking = new Set(); const appsStorageViolations = []; -/** - * Remove rows that duplicate a would-be-unique key, keeping the newest of each. - * - * A recovery strategy for ensureIndex: when a unique build fails because the - * collection already holds rows that violate it, this makes the data conform to - * the invariant the index DECLARES - it deletes duplicates on the key the index - * says must be unique, which is enforcing a contract rather than losing data. - * Only safe where the key IS the row's identity, so it is passed in per build by - * the caller that knows the collection, never applied by default. A rollup whose - * duplicates must be summed rather than dropped (the tampering incident count) - * belongs to its owning service instead - see the note on ensureIndex. - * - * Keeps the newest per group (rows sort by _id, which is time-ordered), honours - * the index's partialFilterExpression so it only touches rows the index covers, - * and returns how many it removed. - * - * @param {object} collection - a mongo collection handle - * @param {object} spec - the index key, e.g. { hash: 1 } - * @param {object} options - the index options (read for partialFilterExpression) - * @returns {Promise} rows removed - */ -async function dedupeByKey(collection, spec, options = {}) { - const groupId = {}; - Object.keys(spec).forEach((key, i) => { groupId[`k${i}`] = `$${key}`; }); - // Held in memory deliberately, and measured rather than assumed: run against a - // live node's collection unioned with itself until every key appeared 16 times - // - 1,030,208 rows, the duplicate state this exists to repair - it finished in - // under 2s without spilling. The $sort adds nothing on top while it stays an - // index walk on _id, which it is for a spec with no partialFilterExpression; - // the first partial index to use this wants re-measuring, because the $match - // ahead of the sort is what would make the sort blocking. allowDiskUse is not - // set: it would take a mongo below 6.0, where the cap errors instead of - // spilling, and the network floor is moving past that. - // - // ids[0] rather than $max: $group does not document that it carries a - // preceding sort into an accumulator, and that non-guarantee is about results - // merged from several sources - this is one standalone mongod. Checked against - // 64,388 real duplicate groups, ids[0] was the newest in all 64,388. - const pipeline = [ - ...(options.partialFilterExpression ? [{ $match: options.partialFilterExpression }] : []), - { $sort: { _id: -1 } }, - { $group: { _id: groupId, ids: { $push: '$_id' } } }, - { $match: { 'ids.1': { $exists: true } } }, - ]; - const groups = await collection.aggregate(pipeline).toArray(); - const toRemove = groups.flatMap((group) => group.ids.slice(1)); - if (!toRemove.length) return 0; - await collection.deleteMany({ _id: { $in: toRemove } }); - return toRemove.length; -} - -/** - * Assert one index, healing the failures that are recoverable. - * - * - a pre-existing index with conflicting OPTIONS (IndexOptionsConflict / - * IndexKeySpecsConflict) is dropped by its real name and recreated; - * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` - * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the - * index rather than running degraded without it; - * - anything else rethrows. - * - * The rethrow is deliberate. Index setup runs in prepareDatabaseSchema, which - * boot runs again every 15 s until it succeeds (untilSucceeds): a TRANSIENT - * failure (mongo mid-election, a slow-disk blip) heals on the next pass instead - * of being skipped until the next reboot, - * and a genuinely UNRECOVERABLE database wedges loudly - which is correct, since - * a node whose DB cannot hold its schema cannot serve apps and appremove would - * not rescue it. The realistic wedge that finding motivated - a unique index - * over rows that already violate it - is repaired above, not hidden. - * - * TRUE NORTH: eventually every collection owns its own schema-prepare - its - * index spec plus whatever dedupe or merge its data needs - the way - * appsRuntimeState.prepareCollection and appTamperingDetectionService already - * do, and boot just invokes those prepare functions. That turns this ~40-call - * imperative block into a set of owned, individually testable units. This - * function is the increment toward it, not the destination; a full move of the - * remaining builds is a separate refactor, out of scope for the PR that added it. - * - * @param {object} collection - a mongo collection handle - * @param {object} spec - the index key - * @param {object} [options] - the index options - * @param {(collection: object, spec: object, options: object) => Promise} [recover] - * run when a unique build is blocked by existing duplicate rows - */ -async function ensureIndex(collection, spec, options = {}, recover = null) { - try { - await collection.createIndex(spec, options); - } catch (err) { - const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); - if (conflict) { - const specKeys = JSON.stringify(spec); - const indexes = await collection.listIndexes().toArray(); - const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); - if (match?.name) { - log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); - await collection.dropIndex(match.name); - } - await collection.createIndex(spec, options); - return; - } - const duplicate = err && (err.code === 11000 || err.codeName === 'DuplicateKey'); - if (duplicate && recover) { - const removed = await recover(collection, spec, options); - log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); - await collection.createIndex(spec, options); - return; - } - throw err; - } -} - -/** - * Assert every index one collection needs, in a single command. - * - * mongo's index build protocol has a fixed cost per BUILD - register, start, - * scan, wait for commit quorum, commit, log - and it does not care that the - * collection is empty. A node asserting its schema one index at a time pays - * that cost 34 times over 14 collections; createIndexes pays it once per - * collection. A node boots faster for it, and the integration harness, where - * ten nodes share one mongod and every database is new, feels it ten times over - * (measured: 938 concurrent builds per ten-node fleet). - * - * The batch is the fast path, not the only one. ensureIndex heals two failures - * that need to be attributed to a single index - an options conflict it drops - * and rebuilds, and a unique build blocked by duplicate rows it repairs through - * the caller's strategy - and a batch rejection does not say which member - * failed. So any error falls back to asserting them one at a time, which is - * exactly the behaviour that existed before this function. - * - * @param {object} collection - a mongo collection handle - * @param {Array} specs - `{ key, ...indexOptions, recover }` per index, - * where `recover` is ours and never reaches mongo - */ -async function ensureIndexes(collection, specs) { - try { - await collection.createIndexes(specs.map((spec) => { - // `recover` is a FluxOS concern; mongo is handed the index model alone - const model = { ...spec }; - delete model.recover; - return model; - })); - return; - } catch (error) { - log.warn(`ensureIndexes - batch of ${specs.length} on ${collection.collectionName} failed (${error.codeName || error.message}); asserting one at a time`); - } - // eslint-disable-next-line no-restricted-syntax - for (const { key, recover = null, ...options } of specs) { - // eslint-disable-next-line no-await-in-loop - await ensureIndex(collection, key, options, recover); - } -} - // How long boot waits before it runs a database step that failed again. const BOOT_STEP_RETRY_MS = 15 * 1000; @@ -305,16 +153,16 @@ async function prepareDatabaseSchema() { log.error(error); } }))); - await ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ { key: { createdAt: 1 }, expireAfterSeconds: 14 * 24 * 60 * 60 }, ]); - await ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ { key: { createdAt: 1 }, expireAfterSeconds: 900 }, ]); - await ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ { key: { createdAt: 1 }, expireAfterSeconds: 900 }, ]); - await ensureIndexes(database.collection(config.database.local.collections.loginSignatures), [ + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.loginSignatures), [ { key: { createdAt: 1 }, expireAfterSeconds: 60 }, { key: { loginPhrase: 1 } }, ]); @@ -322,7 +170,7 @@ async function prepareDatabaseSchema() { // documents expire via lastSeen. The tamper service purges pre-schema // rows at startup, so the detectedAt pair only matters where old code // still writes; drop it once the fleet is past the incident schema. - await ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ { key: { detectedAt: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days { key: { appName: 1, detectedAt: -1 }, name: 'appName_detectedAt' }, { key: { lastSeen: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days @@ -341,7 +189,7 @@ async function prepareDatabaseSchema() { log.info('Preparing temporary database...'); // no need to drop temporary messages const databaseTemp = db.db(config.database.appsglobal.database); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ { key: { receivedAt: 1 }, expireAfterSeconds: tempMsgTtlS }, ]); log.info('Temporary database prepared'); @@ -357,22 +205,22 @@ async function prepareDatabaseSchema() { // we have to create this index again here, as we need it to repair the db. As we were deleting this on every reboot (and it was only created when scannedHeight was 0) // Creating an index that already exists is a no-op - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ - { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dedupeByKey }, + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ + { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dbHelper.dedupeByKey }, { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, ]); await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); // TTL is driven by expireAt (set per-document by store functions). - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, { key: { ip: 1, name: 1 } }, ]); log.info('Flux Apps locations prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dedupeByKey }, + { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dbHelper.dedupeByKey }, { key: { broadcastedAt: 1 } }, { key: { createdAt: 1 } }, // the events of one type still standing, every move among them @@ -381,29 +229,29 @@ async function prepareDatabaseSchema() { { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, ]); log.info('App state events collection prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { broadcastedAt: 1 } }, - { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, + { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dbHelper.dedupeByKey }, ]); log.info('Signed appinstalling broadcasts collection prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, ]); log.info('Flux Apps installing locations prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, { key: { name: 1, hash: 1 }, name: 'query for getting flux app install errors location based on specs name and hash' }, { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, ]); log.info('App installing errors locations prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ { key: { expireAt: 1 }, expireAfterSeconds: 0 }, { key: { broadcastedAt: 1 } }, - { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, + { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dbHelper.dedupeByKey }, ]); log.info('Signed app installing errors broadcasts collection prepared'); @@ -987,11 +835,8 @@ async function startFluxFunctions() { module.exports = { startFluxFunctions, - ensureIndex, - ensureIndexes, untilSucceeds, prepareDatabaseSchema, - dedupeByKey, // testing exports dosState, }; diff --git a/tests/unit/dbHelperIndexes.test.js b/tests/unit/dbHelperIndexes.test.js new file mode 100644 index 0000000000..10be911079 --- /dev/null +++ b/tests/unit/dbHelperIndexes.test.js @@ -0,0 +1,274 @@ +// ensureIndex is the boot sequence's self-healing index builder. It runs in +// serviceManager's prepareDatabaseSchema, which boot runs again until it succeeds: the +// recoverable failures (an options conflict, or a unique build blocked by +// duplicate rows) are healed here, and anything else rethrows - a transient +// blip heals on the next pass, a genuinely broken DB wedges loudly. dedupeByKey is the recovery strategy a unique build +// passes when its key is the row's identity. + +const chai = require('chai'); +const chaiAsPromised = require('chai-as-promised'); +const sinon = require('sinon'); + +chai.use(chaiAsPromised); +const { expect } = chai; + +const { ensureIndex, ensureIndexes, dedupeByKey } = require('../../ZelBack/src/services/dbHelper'); +const log = require('../../ZelBack/src/lib/log'); + +describe('dbHelper ensureIndex', () => { + let logErrorSpy; + + beforeEach(() => { + logErrorSpy = sinon.spy(log, 'error'); + }); + + afterEach(() => { + sinon.restore(); + }); + + function stubCollection(createIndex) { + return { + collectionName: 'somecollection', + createIndex, + listIndexes: sinon.stub(), + dropIndex: sinon.stub().resolves(), + }; + } + + const duplicateError = () => { + const err = new Error('E11000 duplicate key error collection'); + err.code = 11000; + err.codeName = 'DuplicateKey'; + return err; + }; + + it('builds the index and touches nothing else on success', async () => { + const createIndex = sinon.stub().resolves('spec_1'); + const collection = stubCollection(createIndex); + + await ensureIndex(collection, { spec: 1 }, { unique: true }); + + sinon.assert.calledOnceWithExactly(createIndex, { spec: 1 }, { unique: true }); + sinon.assert.notCalled(collection.dropIndex); + sinon.assert.notCalled(logErrorSpy); + }); + + it('drops a conflicting index by its actual name and recreates', async () => { + const conflict = new Error('Index with name: spec_1 already exists with different options'); + conflict.codeName = 'IndexOptionsConflict'; + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(conflict); + createIndex.onSecondCall().resolves('spec_1'); + const collection = stubCollection(createIndex); + collection.listIndexes.returns({ + toArray: async () => [ + { key: { _id: 1 }, name: '_id_' }, + { key: { spec: 1 }, name: 'oldname' }, + ], + }); + + await ensureIndex(collection, { spec: 1 }, { expireAfterSeconds: 60 }); + + sinon.assert.calledOnceWithExactly(collection.dropIndex, 'oldname'); + sinon.assert.calledTwice(createIndex); + sinon.assert.notCalled(logErrorSpy); + }); + + it('runs the recovery strategy and rebuilds when duplicate rows block a unique index', async () => { + // The node ends up WITH the index, not running degraded without it. + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(duplicateError()); + createIndex.onSecondCall().resolves('hash_1'); + const collection = stubCollection(createIndex); + const recover = sinon.stub().resolves(3); + + await ensureIndex(collection, { hash: 1 }, { unique: true }, recover); + + sinon.assert.calledOnceWithExactly(recover, collection, { hash: 1 }, { unique: true }); + sinon.assert.calledTwice(createIndex); + // recover ran before the rebuild + sinon.assert.callOrder(createIndex, recover, createIndex); + }); + + it('rethrows a duplicate-key failure with no recovery strategy, rather than skipping', async () => { + // A unique build with no strategy is not silently swallowed - it must be + // seen, not hidden until the next reboot. + const createIndex = sinon.stub().rejects(duplicateError()); + const collection = stubCollection(createIndex); + + await expect(ensureIndex(collection, { hash: 1 }, { unique: true })).to.be.rejectedWith('E11000'); + sinon.assert.calledOnce(createIndex); + }); + + it('rethrows a transient failure so the boot retry can heal it', async () => { + // Not swallowed: index setup runs before services start, so the retry loop + // re-runs safely and a transient blip heals rather than being skipped. + const transient = new Error('connection 5 to mongo timed out'); + transient.codeName = 'NetworkTimeout'; + const createIndex = sinon.stub().rejects(transient); + const collection = stubCollection(createIndex); + const recover = sinon.stub().resolves(0); + + await expect(ensureIndex(collection, { spec: 1 }, {}, recover)).to.be.rejectedWith('timed out'); + // recovery is only for the duplicate-key case, not for a transient error + sinon.assert.notCalled(recover); + }); + + it('rethrows if the rebuild after recovery still fails', async () => { + const createIndex = sinon.stub().rejects(duplicateError()); + const collection = stubCollection(createIndex); + const recover = sinon.stub().resolves(1); + + await expect(ensureIndex(collection, { hash: 1 }, { unique: true }, recover)).to.be.rejectedWith('E11000'); + sinon.assert.calledOnce(recover); + sinon.assert.calledTwice(createIndex); + }); +}); + +describe('dbHelper dedupeByKey', () => { + function stubCollection(groups) { + return { + collectionName: 'somecollection', + aggregate: sinon.stub().returns({ toArray: async () => groups }), + deleteMany: sinon.stub().resolves({ deletedCount: 0 }), + }; + } + + it('removes all but the newest per key group and reports the count', async () => { + // ids come newest-first from the pipeline's _id sort, so slice(1) keeps the + // newest and removes the rest. + const collection = stubCollection([ + { _id: { k0: 'a' }, ids: ['new1', 'old1', 'old2'] }, + { _id: { k0: 'b' }, ids: ['new3', 'old3'] }, + ]); + + const removed = await dedupeByKey(collection, { hash: 1 }, { unique: true }); + + expect(removed).to.equal(3); + sinon.assert.calledOnceWithExactly(collection.deleteMany, { _id: { $in: ['old1', 'old2', 'old3'] } }); + }); + + it('builds the group key from every field of the index spec', async () => { + const collection = stubCollection([]); + + await dedupeByKey(collection, { 'data.name': 1, 'data.ip': 1 }, {}); + + const [pipeline] = collection.aggregate.firstCall.args; + const group = pipeline.find((stage) => stage.$group); + expect(group.$group._id).to.deep.equal({ k0: '$data.name', k1: '$data.ip' }); + }); + + it('honours the index partial filter so it only touches covered rows', async () => { + const collection = stubCollection([]); + const partial = { incidentKey: { $exists: true } }; + + await dedupeByKey(collection, { appName: 1, eventType: 1, incidentKey: 1 }, { partialFilterExpression: partial }); + + const [pipeline] = collection.aggregate.firstCall.args; + expect(pipeline[0]).to.deep.equal({ $match: partial }); + }); + + it('deletes nothing when there are no duplicates', async () => { + const collection = stubCollection([]); + + const removed = await dedupeByKey(collection, { hash: 1 }, {}); + + expect(removed).to.equal(0); + sinon.assert.notCalled(collection.deleteMany); + }); +}); + +// ensureIndexes is the same guarantee taken in one command. mongo pays a fixed +// cost per index BUILD whatever the data size, so a node asserting its schema +// one index at a time pays it 34 times over 14 collections - and in the +// integration harness, where ten nodes share one mongod and every database is +// new, ten nodes do it at once (measured: 938 builds per fleet boot). The batch +// is the fast path only: anything that fails falls back to ensureIndex, because +// the healing above has to know WHICH index failed and a batch rejection does +// not say. +describe('dbHelper ensureIndexes', () => { + afterEach(() => { + sinon.restore(); + }); + + function stubCollection({ createIndexes, createIndex } = {}) { + return { + collectionName: 'somecollection', + createIndexes: createIndexes || sinon.stub().resolves(['a_1']), + createIndex: createIndex || sinon.stub().resolves('a_1'), + listIndexes: sinon.stub(), + dropIndex: sinon.stub().resolves(), + }; + } + + it('asserts every index in ONE command, not one command per index', async () => { + const collection = stubCollection(); + + await ensureIndexes(collection, [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'by name' }, + { key: { ip: 1, name: 1 } }, + ]); + + sinon.assert.calledOnce(collection.createIndexes); + sinon.assert.notCalled(collection.createIndex); + expect(collection.createIndexes.firstCall.args[0]).to.deep.equal([ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'by name' }, + { key: { ip: 1, name: 1 } }, + ]); + }); + + it('never hands mongo the recovery strategy, which is ours', async () => { + const collection = stubCollection(); + + await ensureIndexes(collection, [ + { key: { hash: 1 }, unique: true, recover: dedupeByKey }, + ]); + + const [models] = collection.createIndexes.firstCall.args; + expect(models).to.deep.equal([{ key: { hash: 1 }, unique: true }]); + expect(models[0]).to.not.have.property('recover'); + }); + + it('falls back to one at a time when the batch is refused, and still heals', async () => { + const duplicate = new Error('E11000 duplicate key error collection'); + duplicate.code = 11000; + duplicate.codeName = 'DuplicateKey'; + + const createIndex = sinon.stub(); + createIndex.onFirstCall().resolves('expireAt_1'); + createIndex.onSecondCall().rejects(duplicate); + createIndex.onThirdCall().resolves('hash_1'); + + const collection = stubCollection({ + createIndexes: sinon.stub().rejects(duplicate), + createIndex, + }); + const recover = sinon.stub().resolves(2); + + await ensureIndexes(collection, [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { hash: 1 }, unique: true, recover }, + ]); + + // one per index, with the recovery run between the failed build and its retry + sinon.assert.calledThrice(createIndex); + sinon.assert.calledOnce(recover); + sinon.assert.calledWithExactly(createIndex.firstCall, { expireAt: 1 }, { expireAfterSeconds: 0 }); + sinon.assert.calledWithExactly(createIndex.secondCall, { hash: 1 }, { unique: true }); + sinon.assert.calledWithExactly(createIndex.thirdCall, { hash: 1 }, { unique: true }); + }); + + it('still rethrows what the per-index path cannot heal', async () => { + const fatal = new Error('not authorized on db to execute command'); + fatal.codeName = 'Unauthorized'; + + const collection = stubCollection({ + createIndexes: sinon.stub().rejects(fatal), + createIndex: sinon.stub().rejects(fatal), + }); + + await expect(ensureIndexes(collection, [{ key: { a: 1 } }])).to.be.rejectedWith('not authorized'); + }); +}); diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index 860ea1fb7c..af7d7ba31b 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -1,10 +1,3 @@ -// ensureIndex is the boot sequence's self-healing index builder. It runs in -// prepareDatabaseSchema, which boot runs again until it succeeds: the -// recoverable failures (an options conflict, or a unique build blocked by -// duplicate rows) are healed here, and anything else rethrows - a transient -// blip heals on the next pass, a genuinely broken DB wedges loudly. dedupeByKey is the recovery strategy a unique build -// passes when its key is the row's identity. - const chai = require('chai'); const chaiAsPromised = require('chai-as-promised'); const sinon = require('sinon'); @@ -13,122 +6,11 @@ chai.use(chaiAsPromised); const { expect } = chai; const { - ensureIndex, ensureIndexes, untilSucceeds, prepareDatabaseSchema, dedupeByKey, dosState, + untilSucceeds, prepareDatabaseSchema, dosState, } = require('../../ZelBack/src/services/serviceManager'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); -const log = require('../../ZelBack/src/lib/log'); - -describe('serviceManager ensureIndex', () => { - let logErrorSpy; - - beforeEach(() => { - logErrorSpy = sinon.spy(log, 'error'); - }); - - afterEach(() => { - sinon.restore(); - }); - - function stubCollection(createIndex) { - return { - collectionName: 'somecollection', - createIndex, - listIndexes: sinon.stub(), - dropIndex: sinon.stub().resolves(), - }; - } - - const duplicateError = () => { - const err = new Error('E11000 duplicate key error collection'); - err.code = 11000; - err.codeName = 'DuplicateKey'; - return err; - }; - - it('builds the index and touches nothing else on success', async () => { - const createIndex = sinon.stub().resolves('spec_1'); - const collection = stubCollection(createIndex); - - await ensureIndex(collection, { spec: 1 }, { unique: true }); - - sinon.assert.calledOnceWithExactly(createIndex, { spec: 1 }, { unique: true }); - sinon.assert.notCalled(collection.dropIndex); - sinon.assert.notCalled(logErrorSpy); - }); - - it('drops a conflicting index by its actual name and recreates', async () => { - const conflict = new Error('Index with name: spec_1 already exists with different options'); - conflict.codeName = 'IndexOptionsConflict'; - const createIndex = sinon.stub(); - createIndex.onFirstCall().rejects(conflict); - createIndex.onSecondCall().resolves('spec_1'); - const collection = stubCollection(createIndex); - collection.listIndexes.returns({ - toArray: async () => [ - { key: { _id: 1 }, name: '_id_' }, - { key: { spec: 1 }, name: 'oldname' }, - ], - }); - - await ensureIndex(collection, { spec: 1 }, { expireAfterSeconds: 60 }); - - sinon.assert.calledOnceWithExactly(collection.dropIndex, 'oldname'); - sinon.assert.calledTwice(createIndex); - sinon.assert.notCalled(logErrorSpy); - }); - - it('runs the recovery strategy and rebuilds when duplicate rows block a unique index', async () => { - // The node ends up WITH the index, not running degraded without it. - const createIndex = sinon.stub(); - createIndex.onFirstCall().rejects(duplicateError()); - createIndex.onSecondCall().resolves('hash_1'); - const collection = stubCollection(createIndex); - const recover = sinon.stub().resolves(3); - - await ensureIndex(collection, { hash: 1 }, { unique: true }, recover); - - sinon.assert.calledOnceWithExactly(recover, collection, { hash: 1 }, { unique: true }); - sinon.assert.calledTwice(createIndex); - // recover ran before the rebuild - sinon.assert.callOrder(createIndex, recover, createIndex); - }); - - it('rethrows a duplicate-key failure with no recovery strategy, rather than skipping', async () => { - // A unique build with no strategy is not silently swallowed - it must be - // seen, not hidden until the next reboot. - const createIndex = sinon.stub().rejects(duplicateError()); - const collection = stubCollection(createIndex); - - await expect(ensureIndex(collection, { hash: 1 }, { unique: true })).to.be.rejectedWith('E11000'); - sinon.assert.calledOnce(createIndex); - }); - - it('rethrows a transient failure so the boot retry can heal it', async () => { - // Not swallowed: index setup runs before services start, so the retry loop - // re-runs safely and a transient blip heals rather than being skipped. - const transient = new Error('connection 5 to mongo timed out'); - transient.codeName = 'NetworkTimeout'; - const createIndex = sinon.stub().rejects(transient); - const collection = stubCollection(createIndex); - const recover = sinon.stub().resolves(0); - - await expect(ensureIndex(collection, { spec: 1 }, {}, recover)).to.be.rejectedWith('timed out'); - // recovery is only for the duplicate-key case, not for a transient error - sinon.assert.notCalled(recover); - }); - - it('rethrows if the rebuild after recovery still fails', async () => { - const createIndex = sinon.stub().rejects(duplicateError()); - const collection = stubCollection(createIndex); - const recover = sinon.stub().resolves(1); - - await expect(ensureIndex(collection, { hash: 1 }, { unique: true }, recover)).to.be.rejectedWith('E11000'); - sinon.assert.calledOnce(recover); - sinon.assert.calledTwice(createIndex); - }); -}); describe('serviceManager untilSucceeds', () => { afterEach(() => sinon.restore()); @@ -160,153 +42,6 @@ describe('serviceManager prepareDatabaseSchema', () => { }); }); -describe('serviceManager dedupeByKey', () => { - function stubCollection(groups) { - return { - collectionName: 'somecollection', - aggregate: sinon.stub().returns({ toArray: async () => groups }), - deleteMany: sinon.stub().resolves({ deletedCount: 0 }), - }; - } - - it('removes all but the newest per key group and reports the count', async () => { - // ids come newest-first from the pipeline's _id sort, so slice(1) keeps the - // newest and removes the rest. - const collection = stubCollection([ - { _id: { k0: 'a' }, ids: ['new1', 'old1', 'old2'] }, - { _id: { k0: 'b' }, ids: ['new3', 'old3'] }, - ]); - - const removed = await dedupeByKey(collection, { hash: 1 }, { unique: true }); - - expect(removed).to.equal(3); - sinon.assert.calledOnceWithExactly(collection.deleteMany, { _id: { $in: ['old1', 'old2', 'old3'] } }); - }); - - it('builds the group key from every field of the index spec', async () => { - const collection = stubCollection([]); - - await dedupeByKey(collection, { 'data.name': 1, 'data.ip': 1 }, {}); - - const [pipeline] = collection.aggregate.firstCall.args; - const group = pipeline.find((stage) => stage.$group); - expect(group.$group._id).to.deep.equal({ k0: '$data.name', k1: '$data.ip' }); - }); - - it('honours the index partial filter so it only touches covered rows', async () => { - const collection = stubCollection([]); - const partial = { incidentKey: { $exists: true } }; - - await dedupeByKey(collection, { appName: 1, eventType: 1, incidentKey: 1 }, { partialFilterExpression: partial }); - - const [pipeline] = collection.aggregate.firstCall.args; - expect(pipeline[0]).to.deep.equal({ $match: partial }); - }); - - it('deletes nothing when there are no duplicates', async () => { - const collection = stubCollection([]); - - const removed = await dedupeByKey(collection, { hash: 1 }, {}); - - expect(removed).to.equal(0); - sinon.assert.notCalled(collection.deleteMany); - }); -}); -// ensureIndexes is the same guarantee taken in one command. mongo pays a fixed -// cost per index BUILD whatever the data size, so a node asserting its schema -// one index at a time pays it 34 times over 14 collections - and in the -// integration harness, where ten nodes share one mongod and every database is -// new, ten nodes do it at once (measured: 938 builds per fleet boot). The batch -// is the fast path only: anything that fails falls back to ensureIndex, because -// the healing above has to know WHICH index failed and a batch rejection does -// not say. -describe('serviceManager ensureIndexes', () => { - afterEach(() => { - sinon.restore(); - }); - - function stubCollection({ createIndexes, createIndex } = {}) { - return { - collectionName: 'somecollection', - createIndexes: createIndexes || sinon.stub().resolves(['a_1']), - createIndex: createIndex || sinon.stub().resolves('a_1'), - listIndexes: sinon.stub(), - dropIndex: sinon.stub().resolves(), - }; - } - - it('asserts every index in ONE command, not one command per index', async () => { - const collection = stubCollection(); - - await ensureIndexes(collection, [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'by name' }, - { key: { ip: 1, name: 1 } }, - ]); - - sinon.assert.calledOnce(collection.createIndexes); - sinon.assert.notCalled(collection.createIndex); - expect(collection.createIndexes.firstCall.args[0]).to.deep.equal([ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'by name' }, - { key: { ip: 1, name: 1 } }, - ]); - }); - - it('never hands mongo the recovery strategy, which is ours', async () => { - const collection = stubCollection(); - - await ensureIndexes(collection, [ - { key: { hash: 1 }, unique: true, recover: dedupeByKey }, - ]); - - const [models] = collection.createIndexes.firstCall.args; - expect(models).to.deep.equal([{ key: { hash: 1 }, unique: true }]); - expect(models[0]).to.not.have.property('recover'); - }); - - it('falls back to one at a time when the batch is refused, and still heals', async () => { - const duplicate = new Error('E11000 duplicate key error collection'); - duplicate.code = 11000; - duplicate.codeName = 'DuplicateKey'; - - const createIndex = sinon.stub(); - createIndex.onFirstCall().resolves('expireAt_1'); - createIndex.onSecondCall().rejects(duplicate); - createIndex.onThirdCall().resolves('hash_1'); - - const collection = stubCollection({ - createIndexes: sinon.stub().rejects(duplicate), - createIndex, - }); - const recover = sinon.stub().resolves(2); - - await ensureIndexes(collection, [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { hash: 1 }, unique: true, recover }, - ]); - - // one per index, with the recovery run between the failed build and its retry - sinon.assert.calledThrice(createIndex); - sinon.assert.calledOnce(recover); - sinon.assert.calledWithExactly(createIndex.firstCall, { expireAt: 1 }, { expireAfterSeconds: 0 }); - sinon.assert.calledWithExactly(createIndex.secondCall, { hash: 1 }, { unique: true }); - sinon.assert.calledWithExactly(createIndex.thirdCall, { hash: 1 }, { unique: true }); - }); - - it('still rethrows what the per-index path cannot heal', async () => { - const fatal = new Error('not authorized on db to execute command'); - fatal.codeName = 'Unauthorized'; - - const collection = stubCollection({ - createIndexes: sinon.stub().rejects(fatal), - createIndex: sinon.stub().rejects(fatal), - }); - - await expect(ensureIndexes(collection, [{ key: { a: 1 } }])).to.be.rejectedWith('not authorized'); - }); -}); - describe('serviceManager apps availability DOS state', () => { afterEach(() => { fluxNetworkHelper.setDosMessage(null); From bdd5fd0243959f6a5e36bce60a704363a00adaed Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:47:48 +0100 Subject: [PATCH 084/369] fix(db): the explorer's collections hold their indexes on every node, and a scan from zero waits for them explorerService.prepareCollections declares the indexes of the explorer's collections, and boot asserts them on every start; an index that exists costs a catalog check. A scan from zero empties those collections and runs the same function before it scans, from initiateBlockProcessor and from recoverAndRestart alike, so a failure in either step retries the preparation and never starts a scan without it. The unique app hash index is built over existing duplicates by keeping each hash's earliest payment whose transaction is on the chain, the record a scan keeps. Boot's appsMessages list carries the message indexes, and the registry rebuild those of appsInformation. The location and installing indexes the explorer built in the chain database are gone; boot builds them in the app database. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/explorerService.js | 178 ++++++++++++++---------- ZelBack/src/services/serviceManager.js | 7 + tests/unit/explorerService.test.js | 74 +++++++++- 3 files changed, 186 insertions(+), 73 deletions(-) diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index 5ad9aeab71..4170e8729a 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -748,6 +748,105 @@ async function processBlock(blockHeight, isInsightExplorer) { } } +/** + * Removes every payment record but the one a scan keeps for each app message + * hash, and answers how many it removed: a payment this node already holds + * stands (standingAppPayment), so a scan keeps the earliest, and a record + * whose transaction the chain does not hold is no payment and gives way to one + * that is. The recovery the unique hash index is built with. + * @param {object} collection the appsHashes collection + * @returns {Promise} + */ +async function keepEarliestPayment(collection) { + const groups = await collection.aggregate([ + { $addFields: { offChain: { $cond: [{ $eq: ['$notOnChain', true] }, 1, 0] } } }, + { $sort: { offChain: 1, height: 1, txIndex: 1, _id: 1 } }, + { $group: { _id: '$hash', ids: { $push: '$_id' } } }, + { $match: { 'ids.1': { $exists: true } } }, + ]).toArray(); + const toRemove = groups.flatMap((group) => group.ids.slice(1)); + if (toRemove.length) await collection.deleteMany({ _id: { $in: toRemove } }); + return toRemove.length; +} + +/** + * Asserts every index the explorer's collections need. Boot runs it on every + * start, and a scan from zero runs it after emptying them; an index that + * already exists costs a catalog check. Runs on the maintenance client. + * @returns {Promise} + */ +async function prepareCollections() { + const client = dbHelper.maintenanceConnection() || dbHelper.databaseConnection(); + const database = client.db(config.database.daemon.database); + const databaseUpdates = client.db(config.database.chainparams.database); + await dbHelper.ensureIndexes(database.collection(utxoIndexCollection), [ + { + key: { txid: 1, vout: 1 }, name: 'query for getting utxo', unique: true, recover: dbHelper.dedupeByKey, + }, + { + key: { txid: 1, vout: 1, satoshis: 1 }, name: 'query for getting utxo for zelnode tx', unique: true, recover: dbHelper.dedupeByKey, + }, + { key: { address: 1 }, name: 'query for addresses utxo' }, + { key: { scriptPubKey: 1 }, name: 'query for scriptPubKey utxo' }, + ]); + await dbHelper.ensureIndexes(database.collection(coinbaseFusionIndexCollection), [ + { + key: { txid: 1, vout: 1 }, name: 'query for getting coinbase fusion utxo', unique: true, recover: dbHelper.dedupeByKey, + }, + { + key: { txid: 1, vout: 1, satoshis: 1 }, name: 'query for getting coinbase fusion utxo for zelnode tx', unique: true, recover: dbHelper.dedupeByKey, + }, + { key: { address: 1 }, name: 'query for addresses coinbase fusion utxo' }, + { key: { scriptPubKey: 1 }, name: 'query for scriptPubKey coinbase fusion utxo' }, + ]); + await dbHelper.ensureIndexes(database.collection(addressTransactionIndexCollection), [ + { key: { address: 1 }, name: 'query for addresses transactions' }, + { key: { address: 1, count: 1 }, name: 'query for addresses transactions with count' }, + ]); + await dbHelper.ensureIndexes(database.collection(appsHashesCollection), [ + { key: { txid: 1 }, name: 'query for getting txid' }, + { key: { height: 1 }, name: 'query for getting height' }, + { + key: { hash: 1 }, name: 'query for getting app hash', unique: true, recover: keepEarliestPayment, + }, + { key: { message: 1 }, name: 'query for getting app hashes depending if we have message' }, + ]); + await dbHelper.ensureIndexes(databaseUpdates.collection(chainParamsMessagesCollection), [ + { key: { txid: 1 }, name: 'query for getting txid of some chain parameters update message' }, + { key: { height: 1 }, name: 'query for getting height of some chain parameters update message' }, + { key: { message: 1 }, name: 'query for getting message of some chain parameters update message' }, + { key: { version: 1 }, name: 'query for getting version of some chain parameters update message' }, + ]); +} + +/** + * Empties the explorer's collections and asserts their indexes, for a scan + * from zero. Rejects when either fails, and the scan does not start. + * @returns {Promise} + */ +async function resetExplorerCollections() { + log.info('Preparing daemon collections'); + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.daemon.database); + const databaseUpdates = db.db(config.database.chainparams.database); + const empty = [ + [database, utxoIndexCollection], + [database, addressTransactionIndexCollection], + [database, appsHashesCollection], + [database, coinbaseFusionIndexCollection], + [databaseUpdates, chainParamsMessagesCollection], + ]; + // eslint-disable-next-line no-restricted-syntax + for (const [owner, collection] of empty) { + // eslint-disable-next-line no-await-in-loop + await dbHelper.dropCollection(owner, collection).catch((error) => { + if (error.message !== 'ns not found') throw error; + }); + } + await prepareCollections(); + log.info('Preparation done'); +} + /** * To restore database to specified block height. * @param {number} height Block height. @@ -1173,6 +1272,9 @@ async function recoverAndRestart(restoreDatabase, deepRestore) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.daemon.database); let scannedBlockHeight = await getScannedBlockHeightFromDb(database); + // A scan from zero starts on empty collections holding every index, however + // the attempt before this one ended. + if (scannedBlockHeight === 0) await resetExplorerCollections(); if (scannedBlockHeight !== 0 && restoreDatabase) { const deepRestoreBlocks = config.fluxapps.explorerDeepRestoreBlocks ?? 100; @@ -1235,77 +1337,8 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore) { } const daemonHeight = daemonBlockCount.data; - if (scannedBlockHeight === 0) { - log.info('Preparing daemon collections'); - // The index builds run on the maintenance client. - const longDatabase = dbHelper.onMaintenanceClient(database); - const result = await dbHelper.dropCollection(database, utxoIndexCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const resultB = await dbHelper.dropCollection(database, addressTransactionIndexCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const resultD = await dbHelper.dropCollection(database, appsHashesCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const resultFusion = await dbHelper.dropCollection(database, coinbaseFusionIndexCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const databaseUpdates = dbHelper.onMaintenanceClient(db.db(config.database.chainparams.database)); - const resultChainParams = await dbHelper.dropCollection(databaseUpdates, chainParamsMessagesCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - log.info(result, resultB, resultD, resultFusion, resultChainParams); - - await longDatabase.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting utxo', unique: true }); - await longDatabase.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting utxo for zelnode tx', unique: true }); - await longDatabase.collection(utxoIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses utxo' }); - await longDatabase.collection(utxoIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey utxo' }); - await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting coinbase fusion utxo', unique: true }); - await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting coinbase fusion utxo for zelnode tx', unique: true }); - await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses coinbase fusion utxo' }); - await longDatabase.collection(coinbaseFusionIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey coinbase fusion utxo' }); - await longDatabase.collection(addressTransactionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses transactions' }); - await longDatabase.collection(addressTransactionIndexCollection).createIndex({ address: 1, count: 1 }, { name: 'query for addresses transactions with count' }); - await longDatabase.collection(appsHashesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid' }); - await longDatabase.collection(appsHashesCollection).createIndex({ height: 1 }, { name: 'query for getting height' }); - await longDatabase.collection(appsHashesCollection).createIndex({ hash: 1 }, { name: 'query for getting app hash', unique: true }).catch((error) => { - log.error('Expected throw on index creation as of new uniquness. Do not remove this check until all nodes have rebuild apps data'); - log.error(error); - }); - await longDatabase.collection(appsHashesCollection).createIndex({ message: 1 }, { name: 'query for getting app hashes depending if we have message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid of some chain parameters update message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ height: 1 }, { name: 'query for getting height of some chain parameters update message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ message: 1 }, { name: 'query for getting message of some chain parameters update message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ version: 1 }, { name: 'query for getting version of some chain parameters update message' }); - - const databaseGlobal = dbHelper.onMaintenanceClient(db.db(config.database.appsglobal.database)); - log.info('Preparing apps collections'); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ hash: 1 }, { name: 'query for getting zelapp message based on hash', unique: true }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ txid: 1 }, { name: 'query for getting zelapp message based on txid' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ height: 1 }, { name: 'query for getting zelapp message based on height' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.name': 1 }, { name: 'query for getting app message based on zelapp specs name' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.owner': 1 }, { name: 'query for getting app message based on zelapp specs owner' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.repotag': 1 }, { name: 'query for getting app message based on image' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.version': 1 }, { name: 'query for getting app message based on version' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.nodes': 1 }, { name: 'query for getting app message based on nodes' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ name: 1 }, { name: 'query for getting zelapp based on zelapp specs name' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ owner: 1 }, { name: 'query for getting zelapp based on zelapp specs owner' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ repotag: 1 }, { name: 'query for getting zelapp based on image' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ height: 1 }, { name: 'query for getting zelapp based on last height update' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ hash: 1 }, { name: 'query for getting zelapp based on last hash' }); - await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp location based on zelapp specs name' }); - await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ hash: 1 }, { name: 'query for getting zelapp location based on zelapp hash' }); - await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ ip: 1 }, { name: 'query for getting zelapp location based on ip' }); - await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting app based on ip and name' }); - await longDatabase.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1, broadcastedAt: 1 }, { name: 'query for getting app to ensure we possess a message' }); - await longDatabase.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp install location based on zelapp specs name' }); - await longDatabase.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting flux app install location based on specs name and node ip' }); - await longDatabase.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1 }, { name: 'query for getting flux app install errors location based on specs name' }); - await longDatabase.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash' }); - await longDatabase.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1, ip: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash and node ip' }); - log.info('Preparation done'); - } + // A scan from zero starts on empty collections holding every index. + if (scannedBlockHeight === 0) await resetExplorerCollections(); if (daemonHeight > scannedBlockHeight) { if (scannedBlockHeight !== 0 && restoreDatabase) { @@ -1994,6 +2027,9 @@ function setZelAppSpecsMigrationDone(value) { } module.exports = { + prepareCollections, + resetExplorerCollections, + keepEarliestPayment, initiateBlockProcessor, processBlock, reindexExplorer, diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 3a50a0f55b..e9ad5fff40 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -185,6 +185,8 @@ async function prepareDatabaseSchema() { // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, // then enforce one doc per component identifier await appsRuntimeState.prepareCollection(); + // The explorer's collections: their indexes are the explorer's to declare. + await explorerService.prepareCollections(); log.info('Local database prepared'); log.info('Preparing temporary database...'); // no need to drop temporary messages @@ -207,6 +209,11 @@ async function prepareDatabaseSchema() { // Creating an index that already exists is a no-op await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dbHelper.dedupeByKey }, + { key: { txid: 1 }, name: 'query for getting zelapp message based on txid' }, + { key: { height: 1 }, name: 'query for getting zelapp message based on height' }, + { key: { 'appSpecifications.name': 1 }, name: 'query for getting app message based on zelapp specs name' }, + { key: { 'appSpecifications.owner': 1 }, name: 'query for getting app message based on zelapp specs owner' }, + { key: { 'appSpecifications.repotag': 1 }, name: 'query for getting app message based on image' }, { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, ]); diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 55696a181c..df8a2f1fd4 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -2001,7 +2001,6 @@ describe('explorerService tests', () => { sinon.assert.notCalled(logErrorSpy); sinon.assert.calledWithMatch(logInfoSpy, 'Bootstrap: Using address-index fast path'); sinon.assert.calledWithMatch(logInfoSpy, 'Bootstrap complete'); - sinon.assert.calledWithMatch(logInfoSpy, 'Preparing apps collections'); sinon.assert.calledWithMatch(logInfoSpy, 'Preparation done'); }); @@ -2109,7 +2108,6 @@ describe('explorerService tests', () => { sinon.assert.notCalled(logErrorSpy); sinon.assert.calledWithMatch(logInfoSpy, 'Bootstrap: Using address-index fast path'); - sinon.assert.calledWithMatch(logInfoSpy, 'Preparing apps collections'); sinon.assert.calledWithMatch(logInfoSpy, 'Preparation done'); const dropped = dropCollectionStub.getCalls().map((call) => call.args[1]); expect(dropped).to.not.include('zelappslocation'); @@ -2121,6 +2119,78 @@ describe('explorerService tests', () => { }); }); + describe('the explorer\'s collections', () => { + const { appsHashes } = config.database.daemon.collections; + let hashes; + + beforeEach(async () => { + await dbHelper.initiateDB(); + hashes = dbHelper.databaseConnection().db(config.database.daemon.database).collection(appsHashes); + await hashes.drop().catch(() => {}); + }); + + afterEach(async () => { + sinon.restore(); + await hashes.drop().catch(() => {}); + }); + + it('keeps, of each hash\'s payment records, the earliest whose transaction is on the chain', async () => { + await hashes.insertMany([ + { hash: 'paid', txid: 'stale', height: 200, notOnChain: true }, + { hash: 'paid', txid: 'later', height: 400, txIndex: 1 }, + { hash: 'paid', txid: 'first', height: 300, txIndex: 5 }, + { hash: 'paid', txid: 'sameblock', height: 300, txIndex: 7 }, + { hash: 'single', txid: 'only', height: 500 }, + ]); + + const removed = await explorerService.keepEarliestPayment(hashes); + + expect(removed).to.equal(3); + expect((await hashes.find({}, { projection: { _id: 0, txid: 1 } }).sort({ txid: 1 }).toArray()).map((r) => r.txid)).to.deep.equal(['first', 'only']); + }); + + it('builds the unique hash index over records that hold duplicates, keeping the earliest payment', async () => { + // inserted earliest first, so the newest row is the later payment + await hashes.insertMany([ + { hash: 'paid', txid: 'first', height: 300 }, + { hash: 'paid', txid: 'later', height: 400 }, + ]); + + await explorerService.prepareCollections(); + + const unique = (await hashes.indexes()).find((index) => index.name === 'query for getting app hash'); + expect(unique?.unique, 'the unique hash index').to.equal(true); + expect((await hashes.find({ hash: 'paid' }).toArray()).map((r) => r.txid)).to.deep.equal(['first']); + }); + + it('starts no scan from zero until its collections hold their indexes', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves({ generalScannedHeight: 0 }); + sinon.stub(dbHelper, 'countInDatabase').resolves(1); + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); + sinon.stub(daemonServiceBlockchainRpcs, 'getBlockCount').returns({ status: 'success', data: 200000 }); + // a scan that starts stops the processor, so the assertion below reads it + const scan = sinon.stub(daemonServiceBlockchainRpcs, 'getBlock').callsFake(async () => { + explorerService.setBlockProccessingCanContinue(false); + return { status: 'error', data: {} }; + }); + sinon.stub(dbHelper, 'ensureIndexes').rejects(new MongoNetworkTimeoutError('timed out')); + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); + explorerService.setZelAppSpecsMigrationDone(true); + explorerService.setBlockProccessingCanContinue(true); + try { + await explorerService.initiateBlockProcessor(false, false); + await clock.tickAsync(1000); + + sinon.assert.notCalled(scan); + } finally { + explorerService.setBlockProccessingCanContinue(false); + explorerService.setIsInInitiationOfBP(false); + explorerService.setZelAppSpecsMigrationDone(false); + clock.restore(); + } + }); + }); + describe('getAddressUtxos tests', () => { let findInDatabaseStub; let logErrorSpy; From 61bc15128d7f4907aa1cd2c9a83453d6c4499399 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 12:51:12 +0100 Subject: [PATCH 085/369] test(db): the deferred broadcast test waits for the handler's store The handler runs after verification in its own turn and awaits a write before it stores the move, so the test waits for that store instead of reading it the moment the clock has advanced. Co-Authored-By: Claude Opus 5.5 --- tests/unit/fluxCommunication.test.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index a97b059eaa..54b00ebd69 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -498,8 +498,12 @@ describe('fluxCommunication tests', () => { expect(verify.callCount, 'a copy waits with it').to.equal(1); sinon.assert.notCalled(messageStore.storeIPChangedMessage); + // the handler runs after verification, in its own turn: the store is what + // shows the re-dispatched broadcast was handled + const stored = new Promise((resolve) => { messageStore.storeIPChangedMessage.callsFake(async () => { resolve(); return false; }); }); await clock.tickAsync(30 * 1000); await clock.tickAsync(100); + await stored; expect(verify.callCount).to.equal(2); sinon.assert.calledOnce(messageStore.storeIPChangedMessage); From 9fb2c0d846d3b6c9c011ec239a9e1d9611ac6fc3 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:04:56 +0100 Subject: [PATCH 086/369] fix(db): a duplicate payment cleanup moves the hash's stored message onto the payment it keeps keepEarliestPayment moves the stored app message of every hash it cleaned onto the record it kept (moveStoredMessagesToPayments), so no message is left pointing at a payment that is gone, and the registry rebuild at boot writes the app's row from it. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/explorerService.js | 25 ++++++++++++++++++++----- tests/unit/explorerService.test.js | 24 ++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index 4170e8729a..e01a451c22 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -753,7 +753,9 @@ async function processBlock(blockHeight, isInsightExplorer) { * hash, and answers how many it removed: a payment this node already holds * stands (standingAppPayment), so a scan keeps the earliest, and a record * whose transaction the chain does not hold is no payment and gives way to one - * that is. The recovery the unique hash index is built with. + * that is. The stored message of each such hash then moves to the record kept, + * and the registry rebuild at boot writes its app's row from it. The recovery + * the unique hash index is built with. * @param {object} collection the appsHashes collection * @returns {Promise} */ @@ -761,11 +763,24 @@ async function keepEarliestPayment(collection) { const groups = await collection.aggregate([ { $addFields: { offChain: { $cond: [{ $eq: ['$notOnChain', true] }, 1, 0] } } }, { $sort: { offChain: 1, height: 1, txIndex: 1, _id: 1 } }, - { $group: { _id: '$hash', ids: { $push: '$_id' } } }, - { $match: { 'ids.1': { $exists: true } } }, + { + $group: { + _id: '$hash', + rows: { + $push: { + id: '$_id', txid: '$txid', height: '$height', value: '$value', + }, + }, + }, + }, + { $match: { 'rows.1': { $exists: true } } }, ]).toArray(); - const toRemove = groups.flatMap((group) => group.ids.slice(1)); - if (toRemove.length) await collection.deleteMany({ _id: { $in: toRemove } }); + const toRemove = groups.flatMap((group) => group.rows.slice(1).map((row) => row.id)); + if (!toRemove.length) return 0; + await collection.deleteMany({ _id: { $in: toRemove } }); + await messageVerifier.moveStoredMessagesToPayments(groups.map(({ _id: hash, rows: [kept] }) => ({ + hash, txid: kept.txid, height: kept.height, value: kept.value, + }))); return toRemove.length; } diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index df8a2f1fd4..656bf081d3 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -2149,6 +2149,30 @@ describe('explorerService tests', () => { expect((await hashes.find({}, { projection: { _id: 0, txid: 1 } }).sort({ txid: 1 }).toArray()).map((r) => r.txid)).to.deep.equal(['first', 'only']); }); + it('moves the stored message of a hash it kept the earliest payment of onto that payment', async () => { + const messages = dbHelper.databaseConnection().db(config.database.appsglobal.database).collection(config.database.appsglobal.collections.appsMessages); + await messages.deleteMany({ hash: { $in: ['paid', 'single'] } }); + await hashes.insertMany([ + { hash: 'paid', txid: 'first', height: 300, value: 5 }, + { hash: 'paid', txid: 'later', height: 400, value: 7 }, + { hash: 'single', txid: 'only', height: 500, value: 9 }, + ]); + await messages.insertMany([ + { hash: 'paid', txid: 'later', height: 400, valueSat: 7, appSpecifications: { name: 'PaidTwice' } }, + { hash: 'single', txid: 'only', height: 500, valueSat: 9, appSpecifications: { name: 'PaidOnce' } }, + ]); + try { + await explorerService.keepEarliestPayment(hashes); + + const paid = await messages.findOne({ hash: 'paid' }, { projection: { _id: 0, txid: 1, height: 1, valueSat: 1 } }); + expect(paid).to.deep.equal({ txid: 'first', height: 300, valueSat: 5 }); + const single = await messages.findOne({ hash: 'single' }, { projection: { _id: 0, txid: 1, height: 1 } }); + expect(single, 'a hash with one payment is left as it is').to.deep.equal({ txid: 'only', height: 500 }); + } finally { + await messages.deleteMany({ hash: { $in: ['paid', 'single'] } }); + } + }); + it('builds the unique hash index over records that hold duplicates, keeping the earliest payment', async () => { // inserted earliest first, so the newest row is the later payment await hashes.insertMany([ From c3c492252d1ffd8d3d60df26c8d151bb4a8391d4 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:12:14 +0100 Subject: [PATCH 087/369] test(harness): a deferred install decides nothing about whether the node holds the app wait.js classifies every InstallOutcome, and DEFERRED joins BUSY among the outcomes a waiter keeps waiting through: the node's database did not answer, and a later attempt decides. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/utils/installOutcome.js | 4 ++-- test-infra/runner/framework/wait.js | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/utils/installOutcome.js b/ZelBack/src/services/utils/installOutcome.js index d638c3b5ed..81c18e55e8 100644 --- a/ZelBack/src/services/utils/installOutcome.js +++ b/ZelBack/src/services/utils/installOutcome.js @@ -13,8 +13,8 @@ * * Every outcome except BUSY is returned after the install hold is acquired, so each * is a true statement about the app whoever asked for it: concurrent attempts are - * serialised, and the ones that lose get BUSY. BUSY is therefore the only outcome a - * caller must keep waiting through rather than act on. + * serialised, and the ones that lose get BUSY. BUSY and DEFERRED are therefore the + * outcomes a caller must keep waiting through rather than act on. * * Internal only - nothing answers these to a client, so unlike Privilege the values * are ours as well as the names. They are strings rather than booleans so a call diff --git a/test-infra/runner/framework/wait.js b/test-infra/runner/framework/wait.js index 9d6ada4810..73a77cf552 100644 --- a/test-infra/runner/framework/wait.js +++ b/test-infra/runner/framework/wait.js @@ -12,7 +12,7 @@ const { InstallOutcome } = installOutcomeModule; // timeout that names the wrong cause. const HOLDS_THE_APP = new Set([InstallOutcome.INSTALLED, InstallOutcome.ALREADY_INSTALLED]); const LACKS_THE_APP = new Set([InstallOutcome.DECLINED, InstallOutcome.FAILED]); -const DECIDES_NOTHING = new Set([InstallOutcome.BUSY]); +const DECIDES_NOTHING = new Set([InstallOutcome.BUSY, InstallOutcome.DEFERRED]); const unclassified = Object.values(InstallOutcome).filter( (outcome) => !HOLDS_THE_APP.has(outcome) && !LACKS_THE_APP.has(outcome) && !DECIDES_NOTHING.has(outcome), From 2f4e90f750bf7314b61668306f4df15c6d434dbb Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:18:59 +0100 Subject: [PATCH 088/369] test(harness): 3301 asserts a node answers a database read while mongod gives no answer Paused for 300 s, a node answers a read of its registry within 200 s, the pool wait and the read and its retry, rather than waiting for mongod. The canary is a MongoNetworkTimeoutError or MongoWaitQueueTimeoutError in a node's log, which only a call that timed out leaves. Co-Authored-By: Claude Opus 5.5 --- ...that-stops-answering-takes-nothing-down.js | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js index 1dc4595a12..a7cc4ae66e 100644 --- a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -20,6 +20,8 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // is retried once, so 120 s), and every caller of a call that fails leaves what it // cannot read as it is. Paused for longer than a read's whole budget, the fleet: // +// - answers a call that reads the database, with an error, rather than waiting +// for the database: what turns a stall into a failure each caller handles; // - keeps every FluxOS process it had, so nothing exits on a failed call; // - removes no app and announces no removal; // - keeps an operator-stopped component stopped; @@ -29,11 +31,14 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // answers and runs once, rather than starting again from the top. const subnet = getSubnetConfig(); -// Over a read's whole budget: 60 s, and the driver's one retry. -const PAUSE_MS = 150 * 1000; -// What a call that got no answer leaves in a node's log. One of them shows the -// pause reached FluxOS, so the assertions below are about a stall, not a quiet minute. -const DATABASE_GAVE_NO_ANSWER = /MongoNetworkTimeoutError|MongoWaitQueueTimeoutError|MongoServerSelectionError|timed out/; +// Over a read's whole budget with room to tell it from the pause: a read that has +// to wait for a pooled connection first is answered within 60 s for the pool and +// 2 x 60 s for the read and its retry. +const PAUSE_MS = 300 * 1000; +const READ_ANSWERED_WITHIN_MS = 200 * 1000; +// What a call that got no answer leaves in a node's log, and nothing else does. One +// of them shows the calls failed, so the assertions below are about failed calls. +const DATABASE_GAVE_NO_ANSWER = /MongoNetworkTimeoutError|MongoWaitQueueTimeoutError/; async function fluxosPid(client) { return Number((await execInContainer(client.container, 'cat /tmp/fluxos.pid')).stdout.trim()); @@ -74,6 +79,7 @@ describe('a database that stops answering takes nothing down', function () { describe('while the fleet runs', function () { let pidsBefore; let anchors; + let read; before(async function () { this.timeout(PAUSE_MS + 120000); @@ -82,13 +88,24 @@ describe('a database that stops answering takes nothing down', function () { await pauseMongo(env); paused = true; try { - await new Promise((resolve) => { setTimeout(resolve, PAUSE_MS); }); + await new Promise((resolve) => { setTimeout(resolve, 5000); }); + const startedAt = Date.now(); + const reading = env.clients[runningIdx].get(`/apps/appspecifications/${runningName}`) + .then((body) => ({ body, ms: Date.now() - startedAt }), (error) => ({ error, ms: Date.now() - startedAt })); + await new Promise((resolve) => { setTimeout(resolve, PAUSE_MS - 5000); }); + read = await Promise.race([reading, { ms: null }]); + if (read.ms === null) read = { ms: Date.now() - startedAt, unanswered: true }; } finally { await resumeMongo(env); paused = false; } }); + it('answers a call that reads the database, rather than waiting for it', function () { + expect(read.unanswered, `the node gave no answer in ${Math.round(read.ms / 1000)} s while the database gave none`).to.not.equal(true); + expect(read.ms, 'answered within a read\'s budget').to.be.below(READ_ANSWERED_WITHIN_MS); + }); + it('reached FluxOS: a node logged a call the database did not answer', function () { const stalled = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); expect(stalled, 'no node logged a database call that timed out, so nothing below was tested').to.not.be.empty; From 3e986523ca20209b8002a5bbdcb92dd90220ce36 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:37:33 +0100 Subject: [PATCH 089/369] test(harness): 3301 holds FluxOS's commands while mongod answers its heartbeats mongo-control holds the data commands with the failCommand failpoint, which the harness mongod's enableTestCommands allows, while hello and the driver's heartbeats are answered: the shape of a mongod with one thread stuck. Every call waits, and only a bound on the call ends the wait. 3301 holds them for 300 s while the fleet runs and while a node boots, and the boot case checks the boot was held inside its schema step. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/mongo-control.js | 46 ++++++++---- ...that-stops-answering-takes-nothing-down.js | 71 ++++++++++++------- 2 files changed, 77 insertions(+), 40 deletions(-) diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js index a8bb21cb70..dbb4ad00c7 100644 --- a/test-infra/runner/framework/mongo-control.js +++ b/test-infra/runner/framework/mongo-control.js @@ -1,20 +1,40 @@ -import { getContainerRuntimeClient } from 'testcontainers'; +import { MongoClient } from 'mongodb'; +import { getSubnetConfig } from './subnet-config.js'; -// The fleet's one mongod, held still and let go. A paused container keeps its -// sockets open and answers nothing, which is what a mongod whose thread is stuck -// in the kernel looks like from FluxOS: every call waits, none fails. The harness's -// own database client (dbClient) waits with it, so a suite reads nothing through it -// while mongod is paused. +// The fleet's mongod made to hold the commands FluxOS sends, while it goes on +// answering the driver's heartbeats: what a mongod with one thread stuck in the +// kernel looks like from FluxOS. Every data command waits and none fails, and +// the server still reads as healthy, so nothing but a bound on the call itself +// ends the wait. (A paused container answers no heartbeat either, and the driver +// then fails every call in flight by itself.) +// +// The harness mongod runs with enableTestCommands, which the failCommand +// failpoint needs. A command already held when the failpoint is released waits +// out the rest of its hold. The harness's own database client (dbClient) is held +// with FluxOS, so a suite reads nothing through it meanwhile. -async function mongoContainer(env) { - const client = await getContainerRuntimeClient(); - return client.container.dockerode.getContainer(env.containers.mongo.getId()); +const HELD_COMMANDS = [ + 'find', 'aggregate', 'getMore', 'count', 'distinct', + 'insert', 'update', 'delete', 'findAndModify', + 'createIndexes', 'listIndexes', 'dropIndexes', 'drop', 'listCollections', +]; + +async function failpoint(command) { + const client = await MongoClient.connect(`mongodb://${getSubnetConfig().mongo}:27017`, { directConnection: true }); + try { + await client.db('admin').command({ configureFailPoint: 'failCommand', ...command }); + } finally { + await client.close(); + } } -export async function pauseMongo(env) { - await (await mongoContainer(env)).pause(); +export async function holdCommands(holdMs) { + await failpoint({ + mode: 'alwaysOn', + data: { failCommands: HELD_COMMANDS, blockConnection: true, blockTimeMS: holdMs }, + }); } -export async function resumeMongo(env) { - await (await mongoContainer(env)).unpause(); +export async function releaseCommands() { + await failpoint({ mode: 'off' }); } diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js index a7cc4ae66e..004b4faab4 100644 --- a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -10,15 +10,17 @@ import { waitFor, waitForUp, waitForDown, waitForOperatorIntent, waitForBootSettled, assertNoEvent, } from '../framework/wait.js'; import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; -import { pauseMongo, resumeMongo } from '../framework/mongo-control.js'; +import { holdCommands, releaseCommands } from '../framework/mongo-control.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; -// A mongod that stops answering, and comes back. The fleet shares one mongod, and a -// paused container is what a mongod whose thread is stuck in the kernel looks like -// from FluxOS: every call waits and none fails. FluxOS bounds a call at 60 s (a read -// is retried once, so 120 s), and every caller of a call that fails leaves what it -// cannot read as it is. Paused for longer than a read's whole budget, the fleet: +// A mongod that stops answering FluxOS's commands while it goes on answering the +// driver's heartbeats, and then answers again: what a mongod with one thread stuck +// in the kernel looks like from FluxOS (mongo-control). The fleet shares one mongod. +// Every call waits and none fails, and only a bound on the call ends the wait. +// FluxOS bounds a call at 60 s (a read is retried once, so 120 s), and every caller +// of a call that fails leaves what it cannot read as it is. Held for longer than a +// read's whole budget, the fleet: // // - answers a call that reads the database, with an error, rather than waiting // for the database: what turns a stall into a failure each caller handles; @@ -27,14 +29,14 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // - keeps an operator-stopped component stopped; // - announces its apps again once mongod answers. // -// The second case pauses mongod while a node boots: the boot holds until mongod +// The second case holds the commands while a node boots: the boot holds until mongod // answers and runs once, rather than starting again from the top. const subnet = getSubnetConfig(); -// Over a read's whole budget with room to tell it from the pause: a read that has +// Over a read's whole budget with room to tell it from the hold: a read that has // to wait for a pooled connection first is answered within 60 s for the pool and // 2 x 60 s for the read and its retry. -const PAUSE_MS = 300 * 1000; +const HOLD_MS = 300 * 1000; const READ_ANSWERED_WITHIN_MS = 200 * 1000; // What a call that got no answer leaves in a node's log, and nothing else does. One // of them shows the calls failed, so the assertions below are about failed calls. @@ -51,7 +53,7 @@ describe('a database that stops answering takes nothing down', function () { const stoppedName = `e2edbstop${Date.now()}`; let runningIdx; let stoppedIdx; - let paused = false; + let held = false; before(async function () { this.timeout(420000); @@ -72,7 +74,7 @@ describe('a database that stops answering takes nothing down', function () { after(async function () { this.timeout(60000); - if (paused) await resumeMongo(env).catch(() => {}); + if (held) await releaseCommands().catch(() => {}); await env?.teardown(); }); @@ -82,22 +84,22 @@ describe('a database that stops answering takes nothing down', function () { let read; before(async function () { - this.timeout(PAUSE_MS + 120000); + this.timeout(HOLD_MS + 120000); pidsBefore = await Promise.all(env.clients.map(fluxosPid)); anchors = env.clients.map((client) => client.getLastEventId()); - await pauseMongo(env); - paused = true; + await holdCommands(HOLD_MS); + held = true; try { await new Promise((resolve) => { setTimeout(resolve, 5000); }); const startedAt = Date.now(); const reading = env.clients[runningIdx].get(`/apps/appspecifications/${runningName}`) .then((body) => ({ body, ms: Date.now() - startedAt }), (error) => ({ error, ms: Date.now() - startedAt })); - await new Promise((resolve) => { setTimeout(resolve, PAUSE_MS - 5000); }); + await new Promise((resolve) => { setTimeout(resolve, HOLD_MS - 5000); }); read = await Promise.race([reading, { ms: null }]); if (read.ms === null) read = { ms: Date.now() - startedAt, unanswered: true }; } finally { - await resumeMongo(env); - paused = false; + await releaseCommands(); + held = false; } }); @@ -113,7 +115,7 @@ describe('a database that stops answering takes nothing down', function () { it('keeps every FluxOS process it had', async function () { const pidsAfter = await Promise.all(env.clients.map(fluxosPid)); - expect(pidsAfter, 'a node\'s FluxOS exited during the pause').to.deep.equal(pidsBefore); + expect(pidsAfter, 'a node\'s FluxOS exited during the hold').to.deep.equal(pidsBefore); }); it('removes no app and announces no removal', async function () { @@ -157,29 +159,44 @@ describe('a database that stops answering takes nothing down', function () { // boot that started again from the top would log either of them twice. const BEFORE_SCHEMA = /System service initiated/; const AFTER_SCHEMA = /Flux checks operational/; + // The schema phase begun and not finished when the hold ends: the boot was held + // on the database, inside the step boot runs again until it succeeds. + const SCHEMA_BEGUN = /Preparing local database/; + const SCHEMA_FINISHED = /Local database prepared/; let counts; + let atRelease; let bootAnchor; before(async function () { - this.timeout(PAUSE_MS + 300000); + this.timeout(HOLD_MS + 300000); const { container } = env.clients[idx]; // Event ids are seeded from the clock and keep rising across a restart. bootAnchor = env.clients[idx].getLastEventId(); await crashFluxos(container, { hold: true }); - counts = { before: env.nodeLogCount(idx, BEFORE_SCHEMA), after: env.nodeLogCount(idx, AFTER_SCHEMA) }; - await pauseMongo(env); - paused = true; + counts = { + before: env.nodeLogCount(idx, BEFORE_SCHEMA), + after: env.nodeLogCount(idx, AFTER_SCHEMA), + begun: env.nodeLogCount(idx, SCHEMA_BEGUN), + finished: env.nodeLogCount(idx, SCHEMA_FINISHED), + }; + await holdCommands(HOLD_MS); + held = true; try { await releaseFluxos(container); - await new Promise((resolve) => { setTimeout(resolve, PAUSE_MS); }); + await new Promise((resolve) => { setTimeout(resolve, HOLD_MS); }); + atRelease = { + begun: env.nodeLogCount(idx, SCHEMA_BEGUN) - counts.begun, + finished: env.nodeLogCount(idx, SCHEMA_FINISHED) - counts.finished, + }; } finally { - await resumeMongo(env); - paused = false; + await releaseCommands(); + held = false; } }); - it('held the boot while the database gave no answer', function () { - expect(env.nodeHasLog(idx, /Waiting for MongoDB/), 'the boot never waited on the database, so nothing below was tested').to.equal(true); + it('held the boot inside its schema step while the database gave no answer', function () { + expect(atRelease.begun, 'the boot never reached its schema step, so nothing below was tested').to.be.above(0); + expect(atRelease.finished, 'the schema step finished while the database held every command').to.equal(0); }); it('finishes the boot once, after the database answers', async function () { From b24b821aad2145fd07bad042b186d6ce6bb63a12 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 13:57:24 +0100 Subject: [PATCH 090/369] test(harness): 3301 logs each step of the hold as a TAP comment Co-Authored-By: Claude Opus 5.5 --- ...atabase-that-stops-answering-takes-nothing-down.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js index 004b4faab4..f469405d29 100644 --- a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -42,6 +42,11 @@ const READ_ANSWERED_WITHIN_MS = 200 * 1000; // of them shows the calls failed, so the assertions below are about failed calls. const DATABASE_GAVE_NO_ANSWER = /MongoNetworkTimeoutError|MongoWaitQueueTimeoutError/; +// A TAP comment, so a hold that overruns says which step it was in. +function step(what) { + console.log(`# 3301 ${new Date().toISOString()} ${what}`); +} + async function fluxosPid(client) { return Number((await execInContainer(client.container, 'cat /tmp/fluxos.pid')).stdout.trim()); } @@ -85,10 +90,13 @@ describe('a database that stops answering takes nothing down', function () { before(async function () { this.timeout(HOLD_MS + 120000); + step('reading the FluxOS pids'); pidsBefore = await Promise.all(env.clients.map(fluxosPid)); anchors = env.clients.map((client) => client.getLastEventId()); + step('holding the commands'); await holdCommands(HOLD_MS); held = true; + step('held'); try { await new Promise((resolve) => { setTimeout(resolve, 5000); }); const startedAt = Date.now(); @@ -97,9 +105,12 @@ describe('a database that stops answering takes nothing down', function () { await new Promise((resolve) => { setTimeout(resolve, HOLD_MS - 5000); }); read = await Promise.race([reading, { ms: null }]); if (read.ms === null) read = { ms: Date.now() - startedAt, unanswered: true }; + step(`read ${read.unanswered ? 'unanswered' : 'answered'} after ${read.ms} ms`); } finally { + step('releasing the commands'); await releaseCommands(); held = false; + step('released'); } }); From 292db7fb9a42628b3676f764abc176677af6a565 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 14:05:49 +0100 Subject: [PATCH 091/369] test(harness): releasing held commands kills them, so the release completes at once Turning the failCommand failpoint off waits for every command inside it to leave, and a held command stays for the whole hold. releaseCommands kills the held commands until the failpoint is off: their wait is interruptible, and they return Interrupted. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/mongo-control.js | 32 +++++++++++++++----- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js index dbb4ad00c7..a9b11cf3fe 100644 --- a/test-infra/runner/framework/mongo-control.js +++ b/test-infra/runner/framework/mongo-control.js @@ -9,9 +9,8 @@ import { getSubnetConfig } from './subnet-config.js'; // then fails every call in flight by itself.) // // The harness mongod runs with enableTestCommands, which the failCommand -// failpoint needs. A command already held when the failpoint is released waits -// out the rest of its hold. The harness's own database client (dbClient) is held -// with FluxOS, so a suite reads nothing through it meanwhile. +// failpoint needs. The harness's own database client (dbClient) is held with +// FluxOS, so a suite reads nothing through it meanwhile. const HELD_COMMANDS = [ 'find', 'aggregate', 'getMore', 'count', 'distinct', @@ -19,22 +18,39 @@ const HELD_COMMANDS = [ 'createIndexes', 'listIndexes', 'dropIndexes', 'drop', 'listCollections', ]; -async function failpoint(command) { +async function withAdmin(work) { const client = await MongoClient.connect(`mongodb://${getSubnetConfig().mongo}:27017`, { directConnection: true }); try { - await client.db('admin').command({ configureFailPoint: 'failCommand', ...command }); + return await work(client.db('admin')); } finally { await client.close(); } } export async function holdCommands(holdMs) { - await failpoint({ + await withAdmin((admin) => admin.command({ + configureFailPoint: 'failCommand', mode: 'alwaysOn', data: { failCommands: HELD_COMMANDS, blockConnection: true, blockTimeMS: holdMs }, - }); + })); } +// Turning the failpoint off waits for every command inside it to leave, and a held +// command stays for the whole hold. So the held commands are killed until the +// release completes: their wait is interruptible, and they leave at once. export async function releaseCommands() { - await failpoint({ mode: 'off' }); + await withAdmin(async (admin) => { + let released = false; + const off = admin.command({ configureFailPoint: 'failCommand', mode: 'off' }).finally(() => { released = true; }); + while (!released) { + // eslint-disable-next-line no-await-in-loop + const { inprog } = await admin.command({ currentOp: 1, active: true }); + const held = inprog.filter((op) => op.command && HELD_COMMANDS.some((name) => name in op.command)); + // eslint-disable-next-line no-await-in-loop + await Promise.all(held.map((op) => admin.command({ killOp: 1, op: op.opid }).catch(() => {}))); + // eslint-disable-next-line no-await-in-loop + await Promise.race([off.catch(() => {}), new Promise((resolve) => { setTimeout(resolve, 1000); })]); + } + await off; + }); } From 79f247a5a2a1ce292717b3775e5623c90d720feb Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:16:11 +0100 Subject: [PATCH 092/369] refactor(apps): the master/slave election pass returns when done, and startMasterSlaveApps runs it masterSlaveApps is one pass of the election and schedules nothing. startMasterSlaveApps runs a pass at once and the next one masterSlaveIntervalMs after each pass ends, keeps going after a pass throws, and returns stop(). Boot starts the election through it. A test of the pass awaits it and nothing it started outlives the test. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 43 ++++++-- ZelBack/src/services/serviceManager.js | 8 +- tests/unit/advancedWorkflows.test.js | 102 ++++++++++-------- 3 files changed, 100 insertions(+), 53 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index a6cb5f01f1..cdfac378e6 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -5061,12 +5061,13 @@ function withdrawSeedMarks(members, outcome) { } /** - * Manages syncthing master/slave application coordination using FDM services. + * One pass of the syncthing master/slave election, using FDM services. It + * returns when the pass is done and schedules nothing; startMasterSlaveApps + * runs it for the life of the process. * State (the busy lists, the receive-only cache) is read off globalStateParam at * the point of each decision, never taken as separate parameters: the getters - * hand out snapshots, so any list captured at call time is a photograph of that - * moment - and this function is invoked once at boot and re-invokes itself for - * the life of the process. + * hand out snapshots, so any list captured when the loop starts would be a + * photograph of that moment. * @param {object} globalStateParam - Global state module (busy lists, receive-only cache, run flags) * @param {Function} installedApps - Function to get installed apps * @param {Function} listRunningApps - Function to get running apps @@ -5717,11 +5718,40 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-param-reassign globalStateParam.masterSlaveAppsRunning = false; fluxEventBus.count('masterSlave:cycles'); - await serviceHelper.delay(config.fluxapps.masterSlaveIntervalMs ?? 30 * 1000); - masterSlaveApps(globalStateParam, installedApps, listRunningApps, https); } } +/** + * Runs the master/slave election now, and again masterSlaveIntervalMs after + * each pass ends, so one pass runs at a time. + * @param {object} globalStateParam - Global state module (busy lists, receive-only cache, run flags) + * @param {Function} installedApps - Function to get installed apps + * @param {Function} listRunningApps - Function to get running apps + * @param {object} https - HTTPS module + * @returns {{ stop: Function }} stop() schedules no further pass + */ +function startMasterSlaveApps(globalStateParam, installedApps, listRunningApps, https) { + let timer = null; + let stopped = false; + + async function runPassThenSchedule() { + try { + await masterSlaveApps(globalStateParam, installedApps, listRunningApps, https); + } catch (error) { + log.error(`masterSlaveApps: ${error}`); + } + if (!stopped) timer = setTimeout(runPassThenSchedule, config.fluxapps.masterSlaveIntervalMs ?? 30 * 1000); + } + + function stop() { + stopped = true; + clearTimeout(timer); + } + + runPassThenSchedule(); + return { stop }; +} + module.exports = { createAppVolume, softRegisterAppLocally, @@ -5754,5 +5784,6 @@ module.exports = { checkAndRemoveEnterpriseAppsOnNonArcane, forceAppRemovals, masterSlaveApps, + startMasterSlaveApps, appDockerStart, }; diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index e9ad5fff40..88c3002ee5 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -778,10 +778,10 @@ async function startFluxFunctions() { // concurrently rather than after a timed offset. // The election reads the busy lists and the receive-only cache off // globalState itself at each decision; they are not parameters. The - // getters return snapshots and masterSlaveApps re-invokes itself forever, - // so anything captured at this call is frozen at boot and goes quietly - // stale - which is exactly how the backup/restore guard once died. - advancedWorkflows.masterSlaveApps( + // getters return snapshots and the election runs for the life of the + // process, so anything captured at this call is frozen at boot and goes + // quietly stale. + advancedWorkflows.startMasterSlaveApps( globalState, appQueryService.installedApps, appQueryService.listRunningApps, diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 490e332c30..6bf0539f9d 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -681,8 +681,63 @@ describe('advancedWorkflows tests', () => { }); }); - // Note: masterSlaveApps is a recursive function that continuously runs in production. - // These tests use a counter to prevent infinite recursion after the first iteration. + describe('startMasterSlaveApps tests', () => { + const intervalMs = config.fluxapps.masterSlaveIntervalMs; + // A pass on a node that is installing returns at the top, so each one is a + // single count of masterSlave:cycles and nothing else. + const installingNode = () => ({ installationInProgress: true }); + const noApps = () => sinon.stub().resolves({ status: 'success', data: [] }); + let clock; + let cycles; + let loop; + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + cycles = sinon.spy(fluxEventBus, 'count').withArgs('masterSlave:cycles'); + }); + + afterEach(() => { + loop?.stop(); + loop = null; + }); + + it('runs a pass at once, and the next one the interval after it ends', async () => { + loop = advancedWorkflows.startMasterSlaveApps(installingNode(), noApps(), noApps(), https); + await clock.tickAsync(0); + expect(cycles.callCount).to.equal(1); + + await clock.tickAsync(intervalMs - 1); + expect(cycles.callCount).to.equal(1); + await clock.tickAsync(1); + expect(cycles.callCount).to.equal(2); + await clock.tickAsync(intervalMs); + expect(cycles.callCount).to.equal(3); + }); + + it('runs no further pass once stopped', async () => { + loop = advancedWorkflows.startMasterSlaveApps(installingNode(), noApps(), noApps(), https); + await clock.tickAsync(0); + loop.stop(); + + await clock.tickAsync(intervalMs * 3); + expect(cycles.callCount).to.equal(1); + }); + + it('keeps running passes after one throws', async () => { + fluxEventBus.count.restore(); + const count = sinon.stub(fluxEventBus, 'count'); + count.withArgs('masterSlave:cycles').onFirstCall().throws(new Error('count failed')); + const logError = sinon.stub(log, 'error'); + + loop = advancedWorkflows.startMasterSlaveApps(installingNode(), noApps(), noApps(), https); + await clock.tickAsync(0); + expect(logError.calledWithMatch('count failed'), 'the first pass never threw, so this proves nothing').to.equal(true); + + await clock.tickAsync(intervalMs); + expect(count.withArgs('masterSlave:cycles').callCount).to.equal(2); + }); + }); + describe('masterSlaveApps tests', () => { let globalState; let serviceHelperStub; @@ -699,7 +754,6 @@ describe('advancedWorkflows tests', () => { let syncthingStatusStub; let syncthingDevicesStub; let axiosGetStub; - let recursionCounter; // What FDM sends when it IS answering and this app simply has no primary // yet: a success body carrying an empty ips array. test-infra/fdm-stub @@ -709,7 +763,6 @@ describe('advancedWorkflows tests', () => { const fdmNoPrimary = () => ({ data: { status: 'success', data: { ips: [] } } }); beforeEach(() => { - recursionCounter = 0; // every app directory has its volume mounted unless a test says otherwise sinon.stub(volumeService, 'isPathMounted').resolves(true); globalState = require('../../ZelBack/src/services/utils/globalState'); @@ -739,15 +792,8 @@ describe('advancedWorkflows tests', () => { verdict: peerIdentityService.IdentityVerdict.UNVERIFIABLE, status: 404, data: null, mayReadUnsigned: true, }); - // Stub delay to prevent recursive calls - after first call, block recursion - serviceHelperDelayStub = sinon.stub(serviceHelper, 'delay').callsFake(async () => { - recursionCounter += 1; - if (recursionCounter > 1) { - // Prevent recursion by returning a promise that never resolves - return new Promise(() => {}); - } - return Promise.resolve(); - }); + // a wait inside a pass ends at once unless a test says otherwise + serviceHelperDelayStub = sinon.stub(serviceHelper, 'delay').resolves(); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); fluxNetworkHelperStub = sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress'); @@ -987,25 +1033,7 @@ describe('advancedWorkflows tests', () => { data: [{ path: `/root/.flux/ZelApps/${appId}`, type: 'sendreceive' }], }); - // masterSlaveApps re-invokes itself from its own finally, so a single call - // otherwise executes the election twice and every count assertion doubles. - // Let exactly one full pass run: the first delay resolves but arms the - // installation gate so the recursive pass returns at the top, and the second - // delay never resolves so the chain stops there. - let delayCalls = 0; - serviceHelperDelayStub.resetBehavior(); - serviceHelperDelayStub.callsFake(async () => { - delayCalls += 1; - if (delayCalls === 1) { - globalState.installationInProgress = true; - return undefined; - } - return new Promise(() => {}); - }); - return async () => { - delayCalls = 0; - globalState.installationInProgress = false; await advancedWorkflows.masterSlaveApps( globalState, installedApps, listRunningApps, https, ); @@ -3990,19 +4018,7 @@ describe('advancedWorkflows tests', () => { ], }); - let delayCalls = 0; - serviceHelperDelayStub.resetBehavior(); - serviceHelperDelayStub.callsFake(async () => { - delayCalls += 1; - if (delayCalls === 1) { - globalState.installationInProgress = true; - return undefined; - } - return new Promise(() => {}); - }); const runPass = async () => { - delayCalls = 0; - globalState.installationInProgress = false; await advancedWorkflows.masterSlaveApps( globalState, installedApps, listRunningApps, https, ); From b9d40d1eb88cd29a4882fac8a90b9082fe831301 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:20:04 +0100 Subject: [PATCH 093/369] test(explorer): a node without the insight index fails the block when its soft fork cannot be written Co-Authored-By: Claude Opus 5.5 --- tests/unit/explorerService.test.js | 38 +++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 656bf081d3..0f3bd944ca 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -2819,7 +2819,7 @@ describe('explorerService tests', () => { expect(outcome, 'the block moved on without its price fork').to.be.instanceOf(MongoNetworkTimeoutError); }); - it('a canary: a soft fork that fails for another reason is skipped, as before', async () => { + it('a canary: a soft fork that fails for another reason is skipped', async () => { const updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new Error('unreadable message')); await explorerService.processInsight(softForkBlock(), {}); @@ -2827,6 +2827,42 @@ describe('explorerService tests', () => { sinon.assert.called(updateOne); }); + describe('on a node without the insight index', () => { + // The same soft fork, its sender found through the spent output it names. + const standardSoftForkBlock = () => ({ + ...softForkBlock(), + tx: softForkBlock().tx.map((tx) => ({ ...tx, vin: [{ txid: 'fundingtx', vout: 0 }] })), + }); + const { chainMessages } = config.database.chainparams.collections; + let updateOne; + + beforeEach(() => { + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({ value: { address: config.fluxapps.addressMultisig } }); + sinon.stub(dbHelper, 'insertOneToDatabase').resolves(); + sinon.stub(serviceHelper, 'delay').resolves(); + // The block's address index is written the same way first; only the soft + // fork's own write fails. + updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').resolves(); + }); + + it('fails the block when its soft fork cannot be written', async () => { + updateOne.withArgs(sinon.match.any, chainMessages).rejects(new MongoNetworkTimeoutError('connection timed out')); + + const outcome = await explorerService.processStandard(standardSoftForkBlock(), {}).catch((err) => err); + + sinon.assert.calledWith(updateOne, sinon.match.any, chainMessages); + expect(outcome, 'the block moved on without its price fork').to.be.instanceOf(MongoNetworkTimeoutError); + }); + + it('a canary: a soft fork that fails for another reason is skipped', async () => { + updateOne.withArgs(sinon.match.any, chainMessages).rejects(new Error('unreadable message')); + + await explorerService.processStandard(standardSoftForkBlock(), {}); + + sinon.assert.calledWith(updateOne, sinon.match.any, chainMessages); + }); + }); + it('fails the block when one of its app payments cannot be recorded', async () => { sinon.stub(dbHelper, 'insertManyToDatabase').rejects(new Error('batch failed')); const upsert = sinon.stub(dbHelper, 'updateOneInDatabase'); From 84cc0807325098e5c3e365c5b6c4c77c5de984dc Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:21:18 +0100 Subject: [PATCH 094/369] test(flux): an installing claim whose signed broadcast could not be stored is not relayed handleAppInstallingErrorMessage is exported beside handleAppInstallingMessage. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxCommunication.js | 1 + tests/unit/fluxCommunication.test.js | 66 +++++++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index b0c7685491..95595f86bb 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -2150,6 +2150,7 @@ module.exports = { logSocketsEvery, handleAppRunningMessage, handleAppInstallingMessage, + handleAppInstallingErrorMessage, handleTempSyncResponse, handleAppRunningSyncResponse, handleAppInstallingSyncResponse, diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index 54b00ebd69..b267df29ee 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -3,6 +3,7 @@ const sinon = require('sinon'); const objectHash = require('object-hash'); const WebSocket = require('ws'); const { expect } = require('chai'); +const { MongoNetworkTimeoutError } = require('mongodb'); const log = require('../../ZelBack/src/lib/log'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { FluxTTLCache } = require('../../ZelBack/src/services/utils/cacheManager'); @@ -630,6 +631,71 @@ describe('fluxCommunication tests', () => { }).timeout(5000); }); + describe('the installing claim handlers', () => { + // Each handler records the claim, stores the signed broadcast this node serves + // to peers syncing from it, and relays the claim only once both are recorded. + const handlers = [ + { + name: 'handleAppInstallingMessage', + type: 'fluxappinstalling', + storeClaim: 'storeAppInstallingMessage', + storeSigned: 'storeSignedAppInstallingBroadcast', + }, + { + name: 'handleAppInstallingErrorMessage', + type: 'fluxappinstallingerror', + storeClaim: 'storeAppInstallingErrorMessage', + storeSigned: 'storeSignedAppInstallingErrorBroadcast', + }, + ]; + const fromIp = '127.0.0.5'; + const port = '16127'; + const claim = (type) => ({ + timestamp: Date.now(), + data: { + type, version: 1, name: 'claimedapp', ip: '127.0.0.9:16127', broadcastedAt: Date.now(), + }, + }); + let broadcastHash; + + beforeEach(() => { + broadcastHash = sinon.stub(peerManager, 'broadcastHash'); + sinon.stub(log, 'error'); + }); + + afterEach(() => { + sinon.restore(); + }); + + handlers.forEach(({ + name, type, storeClaim, storeSigned, + }) => { + describe(name, () => { + beforeEach(() => { + sinon.stub(messageStore, storeClaim).resolves(true); + }); + + it('a canary: relays a claim whose signed broadcast is stored', async () => { + sinon.stub(messageStore, storeSigned).resolves(); + const message = claim(type); + + await fluxCommunication[name](message, fromIp, port); + + sinon.assert.calledOnceWithExactly(broadcastHash, objectHash(message.data), `${fromIp}:${port}`); + }); + + it('does not relay a claim whose signed broadcast could not be stored', async () => { + const storeSignedStub = sinon.stub(messageStore, storeSigned).rejects(new MongoNetworkTimeoutError('connection timed out')); + + await fluxCommunication[name](claim(type), fromIp, port); + + sinon.assert.calledOnce(storeSignedStub); + sinon.assert.notCalled(broadcastHash); + }); + }); + }); + }); + describe('connectedPeers tests', () => { const generateResponse = () => { From b6519327e89196154c94e5a16f0cb00f6930d163 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:22:26 +0100 Subject: [PATCH 095/369] refactor(explorer): the explorer's index asserts reach the maintenance client through onMaintenanceClient prepareCollections takes both databases through dbHelper.onMaintenanceClient, which owns the fallback to the main client, and a test holds every index assert to the maintenance client. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/explorerService.js | 6 +++--- tests/unit/explorerService.test.js | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index e01a451c22..a279578f92 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -791,9 +791,9 @@ async function keepEarliestPayment(collection) { * @returns {Promise} */ async function prepareCollections() { - const client = dbHelper.maintenanceConnection() || dbHelper.databaseConnection(); - const database = client.db(config.database.daemon.database); - const databaseUpdates = client.db(config.database.chainparams.database); + const db = dbHelper.databaseConnection(); + const database = dbHelper.onMaintenanceClient(db.db(config.database.daemon.database)); + const databaseUpdates = dbHelper.onMaintenanceClient(db.db(config.database.chainparams.database)); await dbHelper.ensureIndexes(database.collection(utxoIndexCollection), [ { key: { txid: 1, vout: 1 }, name: 'query for getting utxo', unique: true, recover: dbHelper.dedupeByKey, diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 0f3bd944ca..0a4a4f1864 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -2173,6 +2173,16 @@ describe('explorerService tests', () => { } }); + it('asserts every index on the maintenance client', async () => { + const ensureIndexes = sinon.stub(dbHelper, 'ensureIndexes').resolves(); + + await explorerService.prepareCollections(); + + expect(dbHelper.maintenanceConnection(), 'no maintenance client, so this proves nothing').to.not.equal(null); + expect(ensureIndexes.callCount).to.equal(5); + ensureIndexes.getCalls().forEach((call) => expect(call.args[0].client).to.equal(dbHelper.maintenanceConnection())); + }); + it('builds the unique hash index over records that hold duplicates, keeping the earliest payment', async () => { // inserted earliest first, so the newest row is the later payment await hashes.insertMany([ From 83b20cc6d948e844e1cb781ccecabbe78039bd3e Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:23:25 +0100 Subject: [PATCH 096/369] test(apps): forceAppRemovals removes nothing when the apps docker runs cannot be read Co-Authored-By: Claude Opus 5.5 --- tests/unit/advancedWorkflows.test.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 6bf0539f9d..4f4fbef661 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -360,6 +360,17 @@ describe('advancedWorkflows tests', () => { sinon.assert.notCalled(remove); sinon.assert.calledWithMatch(warned, /Unable to read the apps docker runs and the installed apps/); }); + + it('removes nothing when the apps docker runs cannot be read', async () => { + appQueryService.listAllApps.resolves({ status: 'error', data: { message: 'docker did not answer' } }); + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + const warned = sinon.stub(log, 'warn'); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.notCalled(remove); + sinon.assert.calledWithMatch(warned, /Unable to read the apps docker runs and the installed apps/); + }); }); describe('checkAndRemoveEnterpriseAppsOnNonArcane', () => { From 78e719c75a4f5915d4bd78d36724c76002f1d38b Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:57:37 +0100 Subject: [PATCH 097/369] feat(db): the two clients' call timeouts are config keys config.database.callTimeoutMs (60 s) bounds every call on the main client, and config.database.maintenanceCallTimeoutMs (15 min) every call on the maintenance client: one round trip, and the wait for a pooled connection. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 7 ++++++ ZelBack/src/services/dbHelper.js | 34 +++++++++++++----------------- tests/unit/dbHelper.test.js | 8 ++++--- tests/unit/globalconfig/default.js | 7 ++++++ 4 files changed, 34 insertions(+), 22 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index 9dfe5652d4..558ee16e0c 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -43,6 +43,13 @@ module.exports = { database: { url: '127.0.0.1', port: 27017, + // How long one round trip, or the wait for a pooled connection, may take + // before the call fails. callTimeoutMs bounds every call on the main client: + // over four times the slowest healthy round trip seen on the fleet. + // maintenanceCallTimeoutMs bounds the client for the named long operations: + // index builds, the explorer restore's deletes and the registry rebuild. + callTimeoutMs: 60 * 1000, + maintenanceCallTimeoutMs: 15 * 60 * 1000, local: { database: `${dbPrefix}zelfluxlocal`, collections: { diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index abe3a871b1..80de05c10b 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -19,20 +19,17 @@ const mongoUrl = `mongodb://${config.database.url}:${config.database.port}/`; */ let openDBConnection = null; -// The main client's bound on one round trip and on the wait for a pooled -// connection: over four times the slowest healthy round trip measured on the -// fleet. -const MAIN_SOCKET_TIMEOUT_MS = 60 * 1000; - // The client for the named long operations: index builds, the explorer // restore's deletes and the registry rebuild. One of its round trips may take -// up to MAINTENANCE_SOCKET_TIMEOUT_MS; every other call uses the main client. -const MAINTENANCE_SOCKET_TIMEOUT_MS = 15 * 60 * 1000; -const MAINTENANCE_SETTINGS = { - maxPoolSize: 4, - socketTimeoutMS: MAINTENANCE_SOCKET_TIMEOUT_MS, - waitQueueTimeoutMS: MAINTENANCE_SOCKET_TIMEOUT_MS, -}; +// up to config.database.maintenanceCallTimeoutMs; every other call uses the +// main client, bounded by config.database.callTimeoutMs. +function maintenanceSettings() { + return { + maxPoolSize: 4, + socketTimeoutMS: config.database.maintenanceCallTimeoutMs, + waitQueueTimeoutMS: config.database.maintenanceCallTimeoutMs, + }; +} /** * @type {mongodb.MongoClient} @@ -64,7 +61,7 @@ function maintenanceConnection() { /** * `database` on the maintenance client: the same database, whose round trips - * may take up to MAINTENANCE_SOCKET_TIMEOUT_MS. `database` itself while the + * may take up to config.database.maintenanceCallTimeoutMs. `database` itself while the * maintenance client is not connected. * @param {mongodb.Db} database * @returns {mongodb.Db} @@ -98,10 +95,10 @@ async function connectMongoDb(url, settings = {}) { const mongoSettings = { maxPoolSize: 100, // One round trip, and the wait for a pooled connection, end in an error - // after a minute instead of waiting for ever. A read is retried once, so it - // fails after two; a write is not, and its outcome is then unknown. - socketTimeoutMS: MAIN_SOCKET_TIMEOUT_MS, - waitQueueTimeoutMS: MAIN_SOCKET_TIMEOUT_MS, + // after callTimeoutMs instead of waiting for ever. A read is retried once, so + // it fails after two; a write is not, and its outcome is then unknown. + socketTimeoutMS: config.database.callTimeoutMs, + waitQueueTimeoutMS: config.database.callTimeoutMs, ...settings, }; const client = await MongoClient.connect(connectUrl, mongoSettings); @@ -116,7 +113,7 @@ async function connectMongoDb(url, settings = {}) { async function initiateDB() { if (!openDBConnection) { const main = await connectMongoDb(); - maintenanceDBConnection = await connectMongoDb(undefined, MAINTENANCE_SETTINGS).catch(async (error) => { + maintenanceDBConnection = await connectMongoDb(undefined, maintenanceSettings()).catch(async (error) => { await main.close(); throw error; }); @@ -1357,7 +1354,6 @@ module.exports = { dedupeByKey, maintenanceConnection, onMaintenanceClient, - MAINTENANCE_SOCKET_TIMEOUT_MS, aggregateInDatabase, isDatabaseError, appMessagesThatCount, diff --git a/tests/unit/dbHelper.test.js b/tests/unit/dbHelper.test.js index be7ae4db5f..f9b2dd4a1b 100644 --- a/tests/unit/dbHelper.test.js +++ b/tests/unit/dbHelper.test.js @@ -787,7 +787,7 @@ describe('dbHelper tests', () => { const client = () => ({ close: sinon.stub().resolves(), db: sinon.stub().callsFake((name) => ({ databaseName: name, client: 'maintenance' })) }); - it('is connected beside the main client, with fifteen minutes for one round trip where the main client has one', async () => { + it('is connected beside the main client, each bounded by its own configured call timeout', async () => { await dbHelper.closeDbConnection(); const main = client(); const maintenance = client(); @@ -797,8 +797,10 @@ describe('dbHelper tests', () => { await dbHelper.initiateDB(); - expect(connect.firstCall.args[1]).to.include({ maxPoolSize: 100, socketTimeoutMS: 60 * 1000, waitQueueTimeoutMS: 60 * 1000 }); - expect(connect.secondCall.args[1]).to.include({ socketTimeoutMS: 15 * 60 * 1000, waitQueueTimeoutMS: 15 * 60 * 1000 }); + const { callTimeoutMs, maintenanceCallTimeoutMs } = config.database; + expect(callTimeoutMs, 'the two clients share one bound, so this proves nothing').to.not.equal(maintenanceCallTimeoutMs); + expect(connect.firstCall.args[1]).to.include({ maxPoolSize: 100, socketTimeoutMS: callTimeoutMs, waitQueueTimeoutMS: callTimeoutMs }); + expect(connect.secondCall.args[1]).to.include({ socketTimeoutMS: maintenanceCallTimeoutMs, waitQueueTimeoutMS: maintenanceCallTimeoutMs }); expect(dbHelper.databaseConnection()).to.equal(main); expect(dbHelper.maintenanceConnection()).to.equal(maintenance); expect(dbHelper.onMaintenanceClient({ databaseName: 'globalzelappstest' })).to.deep.equal({ databaseName: 'globalzelappstest', client: 'maintenance' }); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 044e4f49ff..3774384b2e 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -29,6 +29,13 @@ module.exports = { database: { url: database, port: 27017, + // How long one round trip, or the wait for a pooled connection, may take + // before the call fails. callTimeoutMs bounds every call on the main client: + // over four times the slowest healthy round trip seen on the fleet. + // maintenanceCallTimeoutMs bounds the client for the named long operations: + // index builds, the explorer restore's deletes and the registry rebuild. + callTimeoutMs: 60 * 1000, + maintenanceCallTimeoutMs: 15 * 60 * 1000, local: { database: 'zelfluxlocaltest', collections: { From 7b2a08051ab93666e4e3bf74d567c2180123519f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 16:58:33 +0100 Subject: [PATCH 098/369] test(harness): 3301 runs its fleet with short database call bounds, so a hold makes the boot's schema step fail The fleet's calls end after 10 s and its long operations after 20 s, and each hold lasts 75 s. The boot case holds a boot whose schema step fails each time its bound runs out, and asserts a call of that boot failed during the hold. Co-Authored-By: Claude Opus 5.5 --- ...that-stops-answering-takes-nothing-down.js | 36 +++++++++++++------ 1 file changed, 25 insertions(+), 11 deletions(-) diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js index f469405d29..d0a7524a26 100644 --- a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -18,9 +18,10 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // driver's heartbeats, and then answers again: what a mongod with one thread stuck // in the kernel looks like from FluxOS (mongo-control). The fleet shares one mongod. // Every call waits and none fails, and only a bound on the call ends the wait. -// FluxOS bounds a call at 60 s (a read is retried once, so 120 s), and every caller -// of a call that fails leaves what it cannot read as it is. Held for longer than a -// read's whole budget, the fleet: +// FluxOS bounds a call at config.database.callTimeoutMs, and a call on the client +// for long operations at maintenanceCallTimeoutMs; a read is retried once. Every +// caller of a call that fails leaves what it cannot read as it is. This fleet runs +// with both bounds short, and held for longer than a read's whole budget, it: // // - answers a call that reads the database, with an error, rather than waiting // for the database: what turns a stall into a failure each caller handles; @@ -29,15 +30,20 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // - keeps an operator-stopped component stopped; // - announces its apps again once mongod answers. // -// The second case holds the commands while a node boots: the boot holds until mongod -// answers and runs once, rather than starting again from the top. +// The second case holds the commands while a node boots. The boot's schema step +// runs on the client for long operations and fails each time its bound runs out; +// boot runs that step again until it succeeds, so the boot runs once rather than +// starting again from the top. const subnet = getSubnetConfig(); -// Over a read's whole budget with room to tell it from the hold: a read that has -// to wait for a pooled connection first is answered within 60 s for the pool and -// 2 x 60 s for the read and its retry. -const HOLD_MS = 300 * 1000; -const READ_ANSWERED_WITHIN_MS = 200 * 1000; +const CALL_TIMEOUT_MS = 10 * 1000; +const MAINTENANCE_CALL_TIMEOUT_MS = 20 * 1000; +// A read that has to wait for a pooled connection first is answered within one +// bound for the pool and two for the read and its retry. +const READ_ANSWERED_WITHIN_MS = 3 * CALL_TIMEOUT_MS + 10 * 1000; +// Over a read's whole budget with room to tell it from the hold, and over three +// of the schema step's bounds. +const HOLD_MS = 75 * 1000; // What a call that got no answer leaves in a node's log, and nothing else does. One // of them shows the calls failed, so the assertions below are about failed calls. const DATABASE_GAVE_NO_ANSWER = /MongoNetworkTimeoutError|MongoWaitQueueTimeoutError/; @@ -62,7 +68,12 @@ describe('a database that stops answering takes nothing down', function () { before(async function () { this.timeout(420000); - env = await createTestEnv({ hookCtx: this, nodes: 10, tickerAutostart: false }); + env = await createTestEnv({ + hookCtx: this, + nodes: 10, + tickerAutostart: false, + configOverrides: { database: { callTimeoutMs: CALL_TIMEOUT_MS, maintenanceCallTimeoutMs: MAINTENANCE_CALL_TIMEOUT_MS } }, + }); await bootAndPeer(env); ({ index: runningIdx } = await seedSimpleApp(env, runningName)); ({ index: stoppedIdx } = await seedSimpleApp(env, stoppedName)); @@ -189,6 +200,7 @@ describe('a database that stops answering takes nothing down', function () { after: env.nodeLogCount(idx, AFTER_SCHEMA), begun: env.nodeLogCount(idx, SCHEMA_BEGUN), finished: env.nodeLogCount(idx, SCHEMA_FINISHED), + noAnswer: env.nodeLogCount(idx, DATABASE_GAVE_NO_ANSWER), }; await holdCommands(HOLD_MS); held = true; @@ -198,6 +210,7 @@ describe('a database that stops answering takes nothing down', function () { atRelease = { begun: env.nodeLogCount(idx, SCHEMA_BEGUN) - counts.begun, finished: env.nodeLogCount(idx, SCHEMA_FINISHED) - counts.finished, + noAnswer: env.nodeLogCount(idx, DATABASE_GAVE_NO_ANSWER) - counts.noAnswer, }; } finally { await releaseCommands(); @@ -207,6 +220,7 @@ describe('a database that stops answering takes nothing down', function () { it('held the boot inside its schema step while the database gave no answer', function () { expect(atRelease.begun, 'the boot never reached its schema step, so nothing below was tested').to.be.above(0); + expect(atRelease.noAnswer, 'no call of the boot failed while it was held, so nothing below was tested').to.be.above(0); expect(atRelease.finished, 'the schema step finished while the database held every command').to.equal(0); }); From 54292108a20c35fe361f57fcb0636a5779ca1781 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 16 Oct 2026 17:05:31 +0100 Subject: [PATCH 099/369] test(harness): 3301 recognises a database call that ran out by the driver's message Co-Authored-By: Claude Opus 5.5 --- ...1-a-database-that-stops-answering-takes-nothing-down.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js index d0a7524a26..2f554b6550 100644 --- a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -44,9 +44,10 @@ const READ_ANSWERED_WITHIN_MS = 3 * CALL_TIMEOUT_MS + 10 * 1000; // Over a read's whole budget with room to tell it from the hold, and over three // of the schema step's bounds. const HOLD_MS = 75 * 1000; -// What a call that got no answer leaves in a node's log, and nothing else does. One -// of them shows the calls failed, so the assertions below are about failed calls. -const DATABASE_GAVE_NO_ANSWER = /MongoNetworkTimeoutError|MongoWaitQueueTimeoutError/; +// The driver's own words for a round trip and for a pooled-connection wait that ran +// out, in a whole error and in its message alike, and nothing else logs them. One of +// them shows the calls failed, so the assertions below are about failed calls. +const DATABASE_GAVE_NO_ANSWER = /connection \d+ to \S+ timed out|Timed out while checking out a connection from connection pool/; // A TAP comment, so a hold that overruns says which step it was in. function step(what) { From 25f12f47a3f771905f75dbed77a38830e8ee02f5 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 07:55:25 +0100 Subject: [PATCH 100/369] test(db): the hash reconstruction tests run against the node's unique hash index The batch case corrects 5,001 records by { hash, txid }. Without the index the explorer gives the collection, each correction scans every record and the case runs past mocha's 2 s timeout; with it the case takes about 1 s. The index is created for each case and dropped after it, so other files sharing the database see the collection as they left it. Co-Authored-By: Claude Opus 5.5 --- tests/unit/registryManager.test.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 7b4c906b0d..b704df4b2e 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1316,15 +1316,22 @@ describe('registryManager tests', () => { type: 'fluxappregister', hash, height: 3000000, timestamp: 1, appSpecifications: { name: `App${hash}`, version: 3, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' }, }); const unresolvedSignals = () => emitSpy.args.filter(([event]) => event === SYNC_EVENTS.HASH_UNRESOLVED).length; + const hashIndex = 'query for getting app hash'; beforeEach(async () => { daemonDb = db.db(config.database.daemon.database); await daemonDb.collection(appsHashes).deleteMany({}); + // the node's own index; without it every correction scans the whole collection + await daemonDb.collection(appsHashes).createIndex({ hash: 1 }, { name: hashIndex, unique: true }); await database.collection(appsMessages).deleteMany({}); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 3000100 } }); emitSpy = sinon.spy(appSyncEvents, 'emit'); }); + afterEach(async () => { + await daemonDb.collection(appsHashes).dropIndex(hashIndex); + }); + it('should signal a hash to fetch when it marks a received message missing', async () => { await daemonDb.collection(appsHashes).insertOne(record('lost', { message: true })); From 4af0a92b0fc69390d9c67f9e31a4bb9a3fa3b6bb Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 09:19:37 +0100 Subject: [PATCH 101/369] perf(db): the boot backfill reads only the payment records without a position The payment records collection is indexed on txIndex. A record without a position is indexed under null, so the backfill's query reads those records and no others, where it read every record on every boot. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/explorerService.js | 2 ++ .../src/services/migrations/paymentPositions.js | 13 +++++++++---- tests/unit/explorerService.test.js | 14 ++++++++++++++ 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index a279578f92..fac89cf273 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -825,6 +825,8 @@ async function prepareCollections() { key: { hash: 1 }, name: 'query for getting app hash', unique: true, recover: keepEarliestPayment, }, { key: { message: 1 }, name: 'query for getting app hashes depending if we have message' }, + // a record without a position is indexed under null, so the boot backfill reads only those + { key: { txIndex: 1 }, name: 'query for getting app hashes without a position' }, ]); await dbHelper.ensureIndexes(databaseUpdates.collection(chainParamsMessagesCollection), [ { key: { txid: 1 }, name: 'query for getting txid of some chain parameters update message' }, diff --git a/ZelBack/src/services/migrations/paymentPositions.js b/ZelBack/src/services/migrations/paymentPositions.js index c63fa4fd94..f4d5418258 100644 --- a/ZelBack/src/services/migrations/paymentPositions.js +++ b/ZelBack/src/services/migrations/paymentPositions.js @@ -10,6 +10,13 @@ const { appPaymentPositions } = require('../appPaymentPositions'); // worth of records never rides on a single call. const BACKFILL_BATCH = 5000; +// The payment records the backfill reads: those without a position. A record +// already marked notOnChain is decided: if its transaction is mined later, the +// explorer records the payment from that block. Read through the collection's +// txIndex index (explorerService.prepareCollections), so a node whose records all +// have positions reads none of them. +const UNPOSITIONED = { txIndex: { $exists: false }, notOnChain: { $ne: true } }; + /** * Gives every app payment record without a position in its block the position * and the height the daemon's address index holds for its transaction: one call @@ -32,10 +39,8 @@ const BACKFILL_BATCH = 5000; async function backfillPaymentPositions() { const database = dbHelper.databaseConnection().db(config.database.daemon.database); const collection = config.database.daemon.collections.appsHashes; - // A record already marked notOnChain is decided: if its transaction is mined - // later, the explorer records the payment from that block. const missing = await dbHelper.findInDatabase( - database, collection, { txIndex: { $exists: false }, notOnChain: { $ne: true } }, { + database, collection, UNPOSITIONED, { projection: { _id: 0, hash: 1, txid: 1, height: 1, value: 1, }, @@ -91,4 +96,4 @@ async function backfillPaymentPositions() { }; } -module.exports = { backfillPaymentPositions, BACKFILL_BATCH }; +module.exports = { backfillPaymentPositions, BACKFILL_BATCH, UNPOSITIONED }; diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 0a4a4f1864..2b19b337b5 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -16,6 +16,7 @@ const { MongoNetworkTimeoutError } = require('mongodb'); const globalState = require('../../ZelBack/src/services/utils/globalState'); const log = require('../../ZelBack/src/lib/log'); const { requireMongo } = require('./dbTestHelper'); +const { UNPOSITIONED } = require('../../ZelBack/src/services/migrations/paymentPositions'); const chai = require('chai'); const chaiAsPromised = require('chai-as-promised'); @@ -2197,6 +2198,19 @@ describe('explorerService tests', () => { expect((await hashes.find({ hash: 'paid' }).toArray()).map((r) => r.txid)).to.deep.equal(['first']); }); + it('lets the boot backfill read only the payment records without a position', async () => { + await hashes.insertMany([ + ...Array.from({ length: 50 }, (_, i) => ({ hash: `placed${i}`, txid: `placed${i}`, height: i, txIndex: 0 })), + { hash: 'unplaced', txid: 'unplaced', height: 60 }, + ]); + + await explorerService.prepareCollections(); + + const { executionStats } = await hashes.find(UNPOSITIONED).explain('executionStats'); + expect(executionStats.nReturned).to.equal(1); + expect(executionStats.totalDocsExamined, 'the backfill read records that have a position').to.equal(1); + }); + it('starts no scan from zero until its collections hold their indexes', async () => { sinon.stub(dbHelper, 'findOneInDatabase').resolves({ generalScannedHeight: 0 }); sinon.stub(dbHelper, 'countInDatabase').resolves(1); From 59e964d7599f00b0748e474f5b332d32435b8395 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 09:58:58 +0100 Subject: [PATCH 102/369] fix(apps): an uninstall whose runtime record or row cannot be deleted fails, and keeps the row The removal reports failed and the installed-app row stays, still describing the app, so the reconciler rebuilds it. Nothing deletes the row later, so a reinstall's row can never be removed by an earlier removal. Co-Authored-By: Claude Opus 5.5 --- .../services/appLifecycle/appUninstaller.js | 43 ++-------------- tests/unit/appUninstaller.test.js | 51 +++++++++---------- 2 files changed, 26 insertions(+), 68 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index f5a2777b53..8fd8b8902c 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -57,39 +57,6 @@ function setOnComponentRemoved(callback) { onComponentRemoved = callback; } -// Waits between attempts at a cleanup step of an uninstall that failed, the -// last repeated until the step succeeds. -const CLEANUP_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; - -/** - * Runs a cleanup step of an uninstall whose containers are already gone, and - * retries it until it succeeds. While a step is outstanding the app is held as - * departing: the reconciler leaves it alone, so a row that outlived its - * containers is never rebuilt on an empty volume, and announcements leave it out. - * @param {string} appName The app the step cleans up after. - * @param {string} what What the step removes, for the log. - * @param {function(): Promise} step - * @param {number} [attempt] Attempts already made. - * @returns {Promise} Whether the step succeeded on this attempt. - */ -async function runCleanupStep(appName, what, step, attempt = 0) { - try { - await step(); - if (attempt > 0) { - globalState.departingApps.leave(appName); - log.info(`${appName}: ${what} removed after ${attempt} failed attempt(s)`); - } - return true; - } catch (error) { - if (attempt === 0) globalState.departingApps.enter(appName); - const delay = CLEANUP_RETRY_MS[Math.min(attempt, CLEANUP_RETRY_MS.length - 1)]; - log.error(`${appName}: ${what} could not be removed (${error.message}); held as departing, retrying in ${Math.round(delay / 1000)}s`); - const timer = setTimeout(runCleanupStep, delay, appName, what, step, attempt + 1); - if (timer.unref) timer.unref(); - return false; - } -} - /** * Stop Syncthing app and clean up cache * @param {string} monitoredName - Monitored app name @@ -1009,7 +976,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // eslint-disable-next-line no-restricted-syntax for (const identifier of removedIdentifiers) { // eslint-disable-next-line no-await-in-loop - await runCleanupStep(appName, `runtime state of ${identifier}`, () => appsRuntimeState.remove(identifier)); + await appsRuntimeState.remove(identifier); if (onComponentRemoved) onComponentRemoved(identifier); } @@ -1124,11 +1091,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen res.write(serviceHelper.ensureString(databaseStatus)); if (res.flush) res.flush(); } - const rowDeleted = await runCleanupStep( - appName, - 'installed-app row', - () => dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection), - ); + await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); // The app is gone for good: nothing reconciles an app with no row, so its // removal records have no reader left. Only this full-uninstall path clears // them - softRemoveAppLocally deletes the row too, but as one step of a @@ -1137,7 +1100,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // tampering. dockerService.clearFluxRemovedContainers(appName); const databaseStatus2 = { - status: rowDeleted ? 'Database cleaned' : 'Database cleanup pending, retrying', + status: 'Database cleaned', }; log.info(databaseStatus2); if (res) { diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index bd4a3472f3..59fc9c6c2d 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -723,39 +723,34 @@ describe('appUninstaller tests', () => { const res = null; // exercised without a response stream describe('a cleanup step that fails after the containers are gone', () => { - let clock; - - afterEach(() => { - clock?.restore(); - clock = undefined; - }); - - it('holds the app as departing until its row is deleted, retrying the delete', async () => { - const deleteRow = sinon.stub(); - deleteRow.onFirstCall().rejects(new Error('connection timed out')); - deleteRow.resolves(); + // The row stays and still describes the app, so the reconciler rebuilds it: a + // removal is never finished later, against a row a reinstall may have written. + it('fails the removal and keeps the row when the row cannot be deleted', async () => { + const deleteRow = sinon.stub().rejects(new Error('connection timed out')); const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); - clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); - - await uninstaller.removeAppLocally('testapp', res, true); - - expect(globalStateStub.departingApps.has('testapp'), 'a row that outlived its containers is left for the reconciler').to.be.true; - await clock.tickAsync(30 * 1000); - sinon.assert.calledTwice(deleteRow); - expect(globalStateStub.departingApps.has('testapp')).to.be.false; + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + + try { + const outcome = await uninstaller.removeAppLocally('testapp', res, true); + await clock.tickAsync(60 * 60 * 1000); + + expect(outcome).to.equal('failed'); + sinon.assert.calledOnce(deleteRow); + } finally { + clock.restore(); + } + expect(globalStateStub.departingApps.has('testapp'), 'left the app held as departing').to.be.false; }); - it('holds the app as departing until its runtime state is removed, retrying the removal', async () => { - const uninstaller = buildUninstaller(v2Spec); - runtimeStateStub.remove.onFirstCall().rejects(new Error('connection timed out')); - clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + it('fails the removal, before the row, when the runtime state cannot be removed', async () => { + const deleteRow = sinon.stub().resolves(); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + runtimeStateStub.remove.rejects(new Error('connection timed out')); - await uninstaller.removeAppLocally('testapp', res, true); + const outcome = await uninstaller.removeAppLocally('testapp', res, true); - expect(globalStateStub.departingApps.has('testapp')).to.be.true; - await clock.tickAsync(30 * 1000); - sinon.assert.calledTwice(runtimeStateStub.remove); - expect(globalStateStub.departingApps.has('testapp')).to.be.false; + expect(outcome).to.equal('failed'); + sinon.assert.notCalled(deleteRow); }); }); From 382d32fec17886a6d476a6db71126f969d2262fa Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:13:42 +0100 Subject: [PATCH 103/369] fix(id): a login page is told its login by the login, not by polling the database verifyLogin publishes each login's outcome under its phrase: the login it wrote, or the error once the phrase is used and the login was not written. The login websocket waits on that outcome, or on the phrase's expiry, instead of reading the database every 500 ms, so a page is never told a login failed while it is being written. A page that connects after the login reads it; one that connects while a login is using its phrase waits for that login's outcome. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/idService.js | 195 +++++++++++++++++------------- tests/unit/idService.test.js | 118 ++++++++++++++++++ 2 files changed, 231 insertions(+), 82 deletions(-) diff --git a/ZelBack/src/services/idService.js b/ZelBack/src/services/idService.js index d59cb6abbe..9ab85c988f 100644 --- a/ZelBack/src/services/idService.js +++ b/ZelBack/src/services/idService.js @@ -1,3 +1,4 @@ +const { EventEmitter } = require('node:events'); const config = require('config'); const qs = require('qs'); const log = require('../lib/log'); @@ -26,6 +27,28 @@ const PRIVILEGE_RESPONSE = Object.freeze({ NONE: 'none', }); +// How long a login phrase is valid for once issued; the activeLoginPhrases TTL +// index removes it then. +const LOGIN_PHRASE_VALID_MS = 15 * 60 * 1000; + +// The outcome of a login, published by verifyLogin under the phrase it used: +// { data } with what the login answered, or { error } once the phrase is used +// and the login was not written. The login websocket waits on it. +const loginOutcomes = new EventEmitter(); +loginOutcomes.setMaxListeners(0); +// The phrases a login has used and not yet published the outcome of. +const loginsInProgress = new Set(); + +/** + * Publish how a login that used its phrase ended. + * @param {string} phrase + * @param {{data?: object, error?: object}} outcome + */ +function publishLoginOutcome(phrase, outcome) { + loginsInProgress.delete(phrase); + loginOutcomes.emit(phrase, outcome); +} + /** * Removes a login phrase, so that it signs in once. * @param {string} phrase @@ -59,7 +82,7 @@ async function loginPhrase(req, res) { const collection = config.database.local.collections.activeLoginPhrases; const timestamp = Date.now(); - const validTill = timestamp + (15 * 60 * 1000); // 15 minutes + const validTill = timestamp + LOGIN_PHRASE_VALID_MS; const phrase = timestamp + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); const newLoginPhrase = { @@ -87,7 +110,7 @@ async function loginPhrase(req, res) { async function emergencyPhrase(req, res) { try { const timestamp = Date.now(); - const validTill = timestamp + (15 * 60 * 1000); // 15 minutes + const validTill = timestamp + LOGIN_PHRASE_VALID_MS; const phrase = timestamp + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); const db = dbHelper.databaseConnection(); @@ -121,6 +144,8 @@ async function verifyLogin(req, res) { body += data; }); req.on('end', async () => { + // set once this request has used the phrase, which no other can then use + let usedPhrase = null; try { const processedBody = serviceHelper.ensureObject(body); const address = processedBody.zelid || processedBody.address; @@ -201,6 +226,8 @@ async function verifyLogin(req, res) { if (!await consumeLoginPhrase(message)) { throw new Error('Signed message is no longer valid. Please request a new one.'); } + usedPhrase = message; + loginsInProgress.add(message); // The signature is the session's credential. The login websocket hands it // to whoever holds the login phrase, so it is kept apart from the login, // in a collection whose TTL index expires it a minute after the login. It @@ -223,6 +250,7 @@ async function verifyLogin(req, res) { expireAt, }; const resMessage = messageHelper.createDataMessage(resData); + publishLoginOutcome(message, { data: resData }); res.json(resMessage); } else { throw new Error('Invalid signature'); @@ -236,6 +264,7 @@ async function verifyLogin(req, res) { } catch (error) { log.error(error); const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); + if (usedPhrase) publishLoginOutcome(usedPhrase, { error: errMessage }); res.json(errMessage); } }); @@ -530,102 +559,104 @@ async function logoutAllUsers(req, res) { } /** - * To check if a login phrase is currently active. The user's Flux ID, login phrase, signature and privilege level are returned if the login phrase is active. + * Answers a login page waiting on its phrase: with the login once the phrase signs + * in, or with why it no longer can. The answer is the outcome verifyLogin + * publishes for the phrase; a page that connects after the login reads it instead. * @param {object} ws Web socket. - * @param {object} req Request. + * @param {string} loginphrase */ async function wsRespondLoginPhrase(ws, loginphrase) { - // console.log(loginphrase) - // respond with object containing address and signature to received message - let connclosed = false; + let settled = false; + let expiry = null; + const noLongerValid = () => messageHelper.createErrorMessage('Signed message is no longer valid. Please request a new one.'); + + function stop() { + settled = true; + loginOutcomes.removeListener(loginphrase, onOutcome); + clearTimeout(expiry); + } + function finish(message, code) { + if (settled) return; + stop(); + try { + ws.send(qs.stringify(message)); + ws.close(code); + } catch (error) { + log.error(error); + } + } + function onOutcome({ data, error }) { + if (data) finish(messageHelper.createDataMessage(data), 4012); + else finish(error); + } + // eslint-disable-next-line no-param-reassign - ws.onclose = (evt) => { - console.log(evt.code); - connclosed = true; - }; + ws.onclose = stop; // eslint-disable-next-line no-param-reassign ws.onerror = (evt) => { log.error(evt.code); - connclosed = true; + stop(); }; - const db = dbHelper.databaseConnection(); + // before the reads, so a login that finishes while they run is not missed + loginOutcomes.on(loginphrase, onOutcome); - const database = db.db(config.database.local.database); - const collection = config.database.local.collections.loggedUsers; + const database = dbHelper.databaseConnection().db(config.database.local.database); const query = { loginPhrase: loginphrase }; - const projection = {}; - // eslint-disable-next-line no-inner-declarations - async function searchDatabase() { + let login; + try { + login = await dbHelper.findOneInDatabase(database, config.database.local.collections.loggedUsers, query, {}); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); + return; + } + if (settled) return; + if (login) { try { - const result = await dbHelper.findOneInDatabase(database, collection, query, projection).catch((error) => { - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - ws.send(qs.stringify(errMessage)); - ws.close(4011); - throw error; - }); - if (result) { - // user is logged, all ok - const loginSignature = await dbHelper.findOneInDatabase(database, config.database.local.collections.loginSignatures, query, projection); - const adminZelid = verificationHelperUtils.nodeOperatorZelid(); - if (!adminZelid) { - throw new Error('Node is still starting and cannot establish privileges yet'); - } - let privilage = PRIVILEGE_RESPONSE.USER; - if (result.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(result.zelid)) { - privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; - } else if (result.zelid === adminZelid) { - privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; - } - const resData = { - message: 'Successfully logged in', - zelid: result.zelid, - loginPhrase: result.loginPhrase, - signature: loginSignature ? loginSignature.signature : undefined, - privilage, - createdAt: result.createdAt, - expireAt: result.expireAt, - }; - const message = messageHelper.createDataMessage(resData); - if (!connclosed) { - try { - ws.send(qs.stringify(message)); - ws.close(4012); - } catch (e) { - log.error(e); - } - } - } else { - // check if this loginPhrase is still active. If so rerun this searching process - const activeLoginPhrasesCollection = config.database.local.collections.activeLoginPhrases; - const resultB = await dbHelper.findOneInDatabase(database, activeLoginPhrasesCollection, query, projection).catch((error) => { - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - ws.send(qs.stringify(errMessage)); - ws.close(4013); - throw error; - }); - if (resultB) { - setTimeout(() => { - if (!connclosed) { - searchDatabase(); - } - }, 500); - } else { - const errMessage = messageHelper.createErrorMessage('Signed message is no longer valid. Please request a new one.'); - if (!connclosed) { - try { - ws.send(qs.stringify(errMessage)); - ws.close(); - } catch (e) { - log.error(e); - } - } - } + const loginSignature = await dbHelper.findOneInDatabase(database, config.database.local.collections.loginSignatures, query, {}); + const adminZelid = verificationHelperUtils.nodeOperatorZelid(); + if (!adminZelid) { + throw new Error('Node is still starting and cannot establish privileges yet'); + } + let privilage = PRIVILEGE_RESPONSE.USER; + if (login.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(login.zelid)) { + privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; + } else if (login.zelid === adminZelid) { + privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } + finish(messageHelper.createDataMessage({ + message: 'Successfully logged in', + zelid: login.zelid, + loginPhrase: login.loginPhrase, + signature: loginSignature ? loginSignature.signature : undefined, + privilage, + createdAt: login.createdAt, + expireAt: login.expireAt, + }), 4012); } catch (error) { log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); } + return; } - searchDatabase(); + + let phrase; + try { + phrase = await dbHelper.findOneInDatabase(database, config.database.local.collections.activeLoginPhrases, query, {}); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4013); + return; + } + if (settled) return; + // a login that has used the phrase and not yet ended publishes its outcome + if (!phrase && !loginsInProgress.has(loginphrase)) { + finish(noLongerValid()); + return; + } + const issuedAt = +loginphrase.substring(0, 13); + expiry = setTimeout(() => finish(noLongerValid()), Math.max(issuedAt + LOGIN_PHRASE_VALID_MS - Date.now(), 0)); + if (expiry.unref) expiry.unref(); } /** diff --git a/tests/unit/idService.test.js b/tests/unit/idService.test.js index c34033ddbc..69f68e187e 100644 --- a/tests/unit/idService.test.js +++ b/tests/unit/idService.test.js @@ -7,6 +7,7 @@ const os = require('os'); const bitcoinMessage = require('bitcoinjs-message'); const proxyquire = require('proxyquire'); const { PassThrough } = require('stream'); +const qs = require('qs'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); @@ -1286,6 +1287,123 @@ describe('idService tests', () => { }); }); + describe('a login page waiting on its phrase', () => { + const zelid = '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg'; + let findOne; + let insert; + let phrase; + let phraseStored; + + const page = () => ({ send: sinon.stub(), close: sinon.stub() }); + const sentMessage = (ws) => qs.parse(ws.send.firstCall.args[0]); + async function login() { + const stream = new PassThrough(); + stream.push(JSON.stringify({ zelid, signature: '1234356asdf', message: phrase })); + stream.end(); + const res = generateResponse(); + await idService.verifyLogin(stream, res); + await serviceHelper.delay(50); + return res; + } + + beforeEach(async () => { + sinon.stub(bitcoinMessage, 'verify').returns(true); + phrase = `${Date.now() - 60000}11111111111111111111111111111`; + phraseStored = true; + findOne = sinon.stub(dbHelper, 'findOneInDatabase'); + findOne.withArgs(sinon.match.any, 'loggedusers').resolves(null); + findOne.withArgs(sinon.match.any, 'activeloginphrases').callsFake(async () => (phraseStored ? { loginPhrase: phrase } : null)); + insert = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').callsFake(async () => { + phraseStored = false; + return { loginPhrase: phrase }; + }); + await dbHelper.initiateDB(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('is told the login the moment it is written, and reads nothing more', async () => { + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + await login(); + + sinon.assert.calledOnce(ws.send); + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + expect(sentMessage(ws).data).to.include({ zelid, loginPhrase: phrase, message: 'Successfully logged in' }); + sinon.assert.calledOnceWithExactly(ws.close, 4012); + sinon.assert.calledOnce(findOne.withArgs(sinon.match.any, 'loggedusers')); + }); + + it('is told the login when it connects after the phrase is used and before the login is written', async () => { + let writeLogin; + insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); + const loggingIn = login(); + await serviceHelper.delay(20); + expect(phraseStored, 'the phrase was not used, so this proves nothing').to.equal(false); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + writeLogin(true); + await loggingIn; + + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('is told the login when it is written while the page is still reading', async () => { + let answerRead; + findOne.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { answerRead = resolve; })); + const ws = page(); + const connecting = idService.wsRespondLoginPhrase(ws, phrase); + + await login(); + answerRead(null); + await connecting; + + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('is told the login failed once its phrase is used and the login cannot be written', async () => { + insert.withArgs(sinon.match.any, 'loggedusers').rejects(new Error('connection timed out')); + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + await login(); + + expect(sentMessage(ws)).to.deep.include({ status: 'error' }); + expect(sentMessage(ws).data.message).to.equal('connection timed out'); + }); + + it('is told its phrase is no longer valid when it expires with no login', async () => { + const clock = sinon.useFakeTimers({ now: Date.now(), toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + try { + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + await clock.tickAsync(14 * 60 * 1000 - 1000); + sinon.assert.notCalled(ws.send); + + await clock.tickAsync(1000); + + expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + } finally { + clock.restore(); + } + }); + + it('is told its phrase is no longer valid when the phrase is gone and no login used it', async () => { + phraseStored = false; + const ws = page(); + + await idService.wsRespondLoginPhrase(ws, phrase); + + expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + }); + }); + describe('wsRespondSignature tests', () => { const generateWebsocket = () => { const ws = {}; From e12398b611d9d4b552608b1d960276ab30adf9d7 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:27:50 +0100 Subject: [PATCH 104/369] fix(db): a duplicate payment cleanup moves the stored message before it removes the duplicates A cleanup that fails part way leaves the duplicates in place, so the next build of the unique hash index finds them and runs the cleanup again. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/explorerService.js | 10 ++++++---- tests/unit/explorerService.test.js | 13 +++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index fac89cf273..3abc877d3c 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -753,9 +753,9 @@ async function processBlock(blockHeight, isInsightExplorer) { * hash, and answers how many it removed: a payment this node already holds * stands (standingAppPayment), so a scan keeps the earliest, and a record * whose transaction the chain does not hold is no payment and gives way to one - * that is. The stored message of each such hash then moves to the record kept, - * and the registry rebuild at boot writes its app's row from it. The recovery - * the unique hash index is built with. + * that is. The stored message of each such hash moves to the record kept before + * the others are removed, and the registry rebuild at boot writes its app's row + * from it. The recovery the unique hash index is built with. * @param {object} collection the appsHashes collection * @returns {Promise} */ @@ -777,10 +777,12 @@ async function keepEarliestPayment(collection) { ]).toArray(); const toRemove = groups.flatMap((group) => group.rows.slice(1).map((row) => row.id)); if (!toRemove.length) return 0; - await collection.deleteMany({ _id: { $in: toRemove } }); + // the messages first: until the duplicates are gone, a failure here is found + // again by the next build of the unique index, which runs this again await messageVerifier.moveStoredMessagesToPayments(groups.map(({ _id: hash, rows: [kept] }) => ({ hash, txid: kept.txid, height: kept.height, value: kept.value, }))); + await collection.deleteMany({ _id: { $in: toRemove } }); return toRemove.length; } diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 2b19b337b5..fd0357b494 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -17,6 +17,7 @@ const globalState = require('../../ZelBack/src/services/utils/globalState'); const log = require('../../ZelBack/src/lib/log'); const { requireMongo } = require('./dbTestHelper'); const { UNPOSITIONED } = require('../../ZelBack/src/services/migrations/paymentPositions'); +const messageVerifier = require('../../ZelBack/src/services/appMessaging/messageVerifier'); const chai = require('chai'); const chaiAsPromised = require('chai-as-promised'); @@ -2150,6 +2151,18 @@ describe('explorerService tests', () => { expect((await hashes.find({}, { projection: { _id: 0, txid: 1 } }).sort({ txid: 1 }).toArray()).map((r) => r.txid)).to.deep.equal(['first', 'only']); }); + it('keeps every duplicate when the stored message cannot be moved, so the next index build recovers them again', async () => { + await hashes.insertMany([ + { hash: 'paid', txid: 'first', height: 300 }, + { hash: 'paid', txid: 'later', height: 400 }, + ]); + sinon.stub(messageVerifier, 'moveStoredMessagesToPayments').rejects(new Error('connection timed out')); + + await expect(explorerService.keepEarliestPayment(hashes)).to.be.rejectedWith('connection timed out'); + + expect(await hashes.countDocuments({ hash: 'paid' })).to.equal(2); + }); + it('moves the stored message of a hash it kept the earliest payment of onto that payment', async () => { const messages = dbHelper.databaseConnection().db(config.database.appsglobal.database).collection(config.database.appsglobal.collections.appsMessages); await messages.deleteMany({ hash: { $in: ['paid', 'single'] } }); From 5aefffd99451f5732a83716ce39213d476423e67 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:32:26 +0100 Subject: [PATCH 105/369] fix(appsync): a peer's sync request ends at the first message whose writes fail A peer answers a state sync in several messages and only the last says done. A message whose writes failed ends the peer's request there, as an unverifiable one does: the rest of its answer is not taken, it is never credited, and another peer is asked. Co-Authored-By: Claude Opus 5.5 --- .../appMessaging/appSyncOrchestrator.js | 25 ++++++++--- ZelBack/src/services/fluxCommunication.js | 32 +++++++++---- ZelBack/src/services/utils/appSyncEvents.js | 1 + test-infra/runner/framework/node-client.js | 1 + tests/unit/appSyncOrchestrator.test.js | 20 +++++++++ tests/unit/fluxCommunication.test.js | 45 +++++++++++++++++++ 6 files changed, 109 insertions(+), 15 deletions(-) diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index b5544d92f2..1c06ff1c06 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -115,6 +115,8 @@ class AppSyncOrchestrator { #ephemeralSyncHandler = null; #ephemeralRefusedHandler = null; #ephemeralUnverifiedHandler = null; + + #ephemeralWriteFailedHandler = null; #ephemeralProgressHandler = null; #hashUnresolvedHandler = null; #broadcastStarted = null; @@ -255,9 +257,12 @@ class AppSyncOrchestrator { this.#ephemeralRefusedHandler = (syncType, peerKey) => this.#onEphemeralSyncRefused(syncType, peerKey); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_REFUSED, this.#ephemeralRefusedHandler); - this.#ephemeralUnverifiedHandler = (peerKey) => this.#onEphemeralSyncUnverified(peerKey); + this.#ephemeralUnverifiedHandler = (peerKey) => this.#endFailedRequest(peerKey, 'unverified', 'sent a response this node could not verify', 'ephemeralSync:peerUnverified'); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_UNVERIFIED, this.#ephemeralUnverifiedHandler); + this.#ephemeralWriteFailedHandler = (peerKey) => this.#endFailedRequest(peerKey, 'writeFailed', 'sent a response this node could not record', 'ephemeralSync:peerWriteFailed'); + appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, this.#ephemeralWriteFailedHandler); + this.#ephemeralProgressHandler = (peerKey) => this.#onEphemeralSyncProgress(peerKey); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_PROGRESS, this.#ephemeralProgressHandler); @@ -569,15 +574,22 @@ class AppSyncOrchestrator { * @param {string} peerKey ip:port of the peer whose response failed. * @returns {void} */ - #onEphemeralSyncUnverified(peerKey) { + /** + * End a peer's request whose answer cannot be taken, and ask another peer. + * @param {string} peerKey + * @param {string} outcome What the request is settled as. + * @param {string} why For the log. + * @param {string} busEvent Published when this ended the request. + */ + #endFailedRequest(peerKey, outcome, why, busEvent) { if (this.#stateSyncComplete) return; if (!peerKey) { - log.error('AppSyncOrchestrator - An unverifiable sync response named no peer, cannot replace it'); + log.error(`AppSyncOrchestrator - A sync response named no peer (${outcome}), cannot replace it`); return; } - if (this.#closeRequest(peerKey, 'unverified')) { - log.warn(`AppSyncOrchestrator - ${peerKey} sent a response this node could not verify, asking another peer`); - fluxEventBus.publish('ephemeralSync:peerUnverified', { peer: peerKey }); + if (this.#closeRequest(peerKey, outcome)) { + log.warn(`AppSyncOrchestrator - ${peerKey} ${why}, asking another peer`); + fluxEventBus.publish(busEvent, { peer: peerKey }); } // Unconditional for the same reason a refusal is: the deficit decides, and // a pool that is already whole asks nobody. @@ -1211,6 +1223,7 @@ class AppSyncOrchestrator { } if (this.#ephemeralUnverifiedHandler) { appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_UNVERIFIED, this.#ephemeralUnverifiedHandler); + appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, this.#ephemeralWriteFailedHandler); } if (this.#hashUnresolvedHandler) { appSyncEvents.removeListener(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 95595f86bb..720c115ce8 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -248,11 +248,13 @@ async function handleAppRunningSyncResponse(message, peerSocket) { }); if (locationWriteFailed) { - // Not credited either: what this peer sent is not all recorded, so it has not answered. - log.warn('handleAppRunningSyncResponse - skipping location pruning, a location write failed'); - } else { - await messageStore.pruneAppRunningLocations(newestByIp); + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleAppRunningSyncResponse - a location write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; } + await messageStore.pruneAppRunningLocations(newestByIp); // Applied after every slice, once every apprunning row in the response is // stored. Each event removes, shortens or carries only the rows broadcast @@ -272,7 +274,7 @@ async function handleAppRunningSyncResponse(message, peerSocket) { } } - if (done && !locationWriteFailed) { + if (done) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apprunning', peerKey); log.info('handleAppRunningSyncResponse - Sync complete'); } @@ -304,8 +306,14 @@ async function handleAppInstallingSyncResponse(message, peerSocket) { log.info(`handleAppInstallingSyncResponse - Stored ${stored.stored} of ${verified.length} verified broadcasts`); fluxEventBus.publish('sync:chunkVerified', { syncType: 'appinstalling', peer: peerKey, verified: verified.length, stored: stored.stored }); }); - // A response whose writes failed is not all recorded: this peer has not answered. - if (done && !writeFailed) { + if (writeFailed) { + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleAppInstallingSyncResponse - a write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; + } + if (done) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'appinstalling', peerKey); log.info('handleAppInstallingSyncResponse - Sync complete'); } @@ -337,8 +345,14 @@ async function handleAppInstallingErrorsSyncResponse(message, peerSocket) { log.info(`handleAppInstallingErrorsSyncResponse - Stored ${stored.stored} of ${verified.length} verified broadcasts`); fluxEventBus.publish('sync:chunkVerified', { syncType: 'apperrors', peer: peerKey, verified: verified.length, stored: stored.stored }); }); - // A response whose writes failed is not all recorded: this peer has not answered. - if (done && !writeFailed) { + if (writeFailed) { + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleAppInstallingErrorsSyncResponse - a write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; + } + if (done) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apperrors', peerKey); log.info('handleAppInstallingErrorsSyncResponse - Sync complete'); } diff --git a/ZelBack/src/services/utils/appSyncEvents.js b/ZelBack/src/services/utils/appSyncEvents.js index aec590dda0..6f3f1a0c21 100644 --- a/ZelBack/src/services/utils/appSyncEvents.js +++ b/ZelBack/src/services/utils/appSyncEvents.js @@ -6,6 +6,7 @@ const EVENTS = Object.freeze({ EPHEMERAL_SYNC_COMPLETE: 'ephemeralSyncComplete', EPHEMERAL_SYNC_REFUSED: 'ephemeralSyncRefused', EPHEMERAL_SYNC_UNVERIFIED: 'ephemeralSyncUnverified', + EPHEMERAL_SYNC_WRITE_FAILED: 'ephemeralSyncWriteFailed', EPHEMERAL_SYNC_PROGRESS: 'ephemeralSyncProgress', SPAWNER_READY: 'spawnerReady', READINESS_LOST: 'readinessLost', diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index d90e705cb2..c0e133aa61 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -317,6 +317,7 @@ export function nodeClient(nodeNum) { 'sync:refused', 'ephemeralSync:peerTimedOut', 'ephemeralSync:peerUnverified', + 'ephemeralSync:peerWriteFailed', 'ephemeralSync:peerDisconnected', 'ephemeralSync:budgetSpent', 'sync:chunkVerified', diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index 3c935a8224..1a54d47335 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -757,6 +757,26 @@ describe('AppSyncOrchestrator', () => { await orchestrator.stop(); }); + it('ends the request and asks another peer when its response cannot be recorded', async () => { + const peers = makeEligiblePeers(3); + const spare = makePeer('10.0.0.9:16127'); + getEligibleSyncPeersStub = sinon.stub().returns(peers); + + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peerEmitter.emit('peerThresholdReached', 12); + await clock.tickAsync(0); + + getEligibleSyncPeersStub.returns([peers[1], peers[2], spare]); + appSyncEvents.emit(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peers[0].key); + await clock.tickAsync(0); + + expect(spare.send.called, 'a peer whose response was not recorded kept its slot').to.equal(true); + expect(orchestrator.isSyncResponseWanted(peers[0]), 'the rest of an unrecorded response was still being taken') + .to.equal(false); + await orchestrator.stop(); + }); + // The response path has the key and always had it. One arriving without a // peer means that path stopped saying, which is a fault to report. it('refuses to act on an unverifiable response that names no peer', async () => { diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index b267df29ee..022fc93720 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -2528,6 +2528,51 @@ describe('fluxCommunication tests', () => { expect(completions).to.deep.equal([{ syncType: 'appinstalling', peerKey: PEER }]); }); + + // An answer comes as several messages and only the last says done, so the + // request ends at the message whose writes failed, wherever it falls. + describe('ends the peer\'s request at a message before its last', () => { + let writeFailures; + const onWriteFailed = (peerKey) => writeFailures.push(peerKey); + const running = () => ({ + type: 'apprunning', + envelope: { version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig' }, + data: { version: 2, ip: '1.2.3.4:16127', broadcastedAt: Date.now(), apps: [{ name: 'a' }] }, + }); + + beforeEach(() => { + writeFailures = []; + appSyncEvents.on(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + }); + + afterEach(() => { + appSyncEvents.removeListener(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + }); + + [ + ['appinstalling', 'storeBatchAppInstallingMessages', 'handleAppInstallingSyncResponse', 'fluxappinstallingsync', broadcast], + ['apperrors', 'storeBatchAppInstallingErrorMessages', 'handleAppInstallingErrorsSyncResponse', 'fluxappinstallingerrorssync', broadcast], + ['apprunning', 'storeBatchAppRunningMessages', 'handleAppRunningSyncResponse', 'fluxapprunningsync', running], + ].forEach(([syncType, store, handler, type, event]) => { + it(`for ${syncType}`, async () => { + sinon.stub(messageStore, store).resolves({ stored: 1, writeFailed: true }); + + await fluxCommunication[handler]({ data: { type, messages: [event()], done: false } }, PEER_SOCKET); + + expect(messageStore[store].called, 'the fixture never reached the store').to.be.true; + expect(writeFailures).to.deep.equal([PEER]); + }); + + it(`a canary: a ${syncType} message written in full ends nothing`, async () => { + sinon.stub(messageStore, store).resolves({ stored: 1, writeFailed: false }); + + await fluxCommunication[handler]({ data: { type, messages: [event()], done: false } }, PEER_SOCKET); + + expect(messageStore[store].called, 'the fixture never reached the store').to.be.true; + expect(writeFailures).to.deep.equal([]); + }); + }); + }); }); describe('a chunk takes its place in the queue when it arrives', () => { From 4178c823ff47bd84bfed75b0111400736c68cf84 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:39:06 +0100 Subject: [PATCH 106/369] feat(appsync): how many events a sync answer carries per message is config fluxapps.syncAnswerBatchSize, 2000 as before. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 2 ++ ZelBack/src/services/fluxCommunicationMessagesSender.js | 4 ++-- tests/unit/globalconfig/default.js | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index 558ee16e0c..eae664a480 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -656,6 +656,8 @@ module.exports = { connectionBackoffMs: [120000, 300000, 600000, 900000], nodeMonitorIntervalMs: 1200000, nodeMonitorRemovalDelayMs: 60000, + // events per message of an answer to a peer's state sync + syncAnswerBatchSize: 2000, // Residential-node staging. The placement hold is immediate and is not // tunable; these pace only the part that moves customer data. residentialCheckIntervalMs: 6 * 60 * 60 * 1000, // re-evaluate the verdict diff --git a/ZelBack/src/services/fluxCommunicationMessagesSender.js b/ZelBack/src/services/fluxCommunicationMessagesSender.js index 6f74309e2c..85543c0493 100644 --- a/ZelBack/src/services/fluxCommunicationMessagesSender.js +++ b/ZelBack/src/services/fluxCommunicationMessagesSender.js @@ -454,7 +454,7 @@ async function respondWithTempMessages(peer, sinceTimestamp = 0) { .find(query, { projection: { _id: 0, receivedAt: 0, expireAt: 0 } }) .sort({ receivedAt: 1 }); - const batchSize = 2000; + const batchSize = config.fluxapps.syncAnswerBatchSize; let batch = []; let total = 0; for await (const msg of cursor) { @@ -515,7 +515,7 @@ async function streamBatchedSync(peer, { sinceTimestamp, collectionName, validit .find(finalQuery, { projection: finalProjection }) .sort({ broadcastedAt: 1 }); - const batchSize = 2000; + const batchSize = config.fluxapps.syncAnswerBatchSize; let batch = []; let total = 0; for await (const doc of cursor) { diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 3774384b2e..9904238210 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -376,6 +376,8 @@ module.exports = { wsHandshakeTimeoutMs: 10000, discoveryConnectionDelayMs: 500, nodeMonitorRemovalDelayMs: 60000, + // events per message of an answer to a peer's state sync + syncAnswerBatchSize: 2000, residentialCheckIntervalMs: 21600000, residentialSettleMs: 86400000, residentialEvacuationIntervalMs: 21600000, From dd70d3d8304b28cec1ce93f81ac7880b329fcd7c Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:39:07 +0100 Subject: [PATCH 107/369] test(harness): 3304, a sync answer the node cannot record is not counted mongo-control.failWrites fails the next writes to one collection of one node, the rest of the fleet untouched, and reports how many it failed. 3304 fails the first location write of a joining node's sync answer and asserts the node asks another peer and never counts the one whose answer it could not record. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/mongo-control.js | 19 ++++ ...r-the-node-cannot-record-is-not-counted.js | 101 ++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js index a9b11cf3fe..ad7075baed 100644 --- a/test-infra/runner/framework/mongo-control.js +++ b/test-infra/runner/framework/mongo-control.js @@ -54,3 +54,22 @@ export async function releaseCommands() { await off; }); } + +// The next `times` writes to one collection of one node fail, as a write the +// database could not complete does, and every other node's go through: each node +// keeps its data in databases of its own (dbClient's prefix). Resolves to a +// release that turns the failure off and resolves to how many writes it failed. +export async function failWrites(nodeNum, database, collection, { times = 1 } = {}) { + const namespace = `node${String(nodeNum).padStart(2, '0')}_${database}.${collection}`; + await withAdmin((admin) => admin.command({ + configureFailPoint: 'failCommand', + mode: { times }, + data: { failCommands: ['insert', 'update'], namespace, errorCode: 91 }, + })); + return async function release() { + // turning a failpoint off answers how many times it fired + const { count } = await withAdmin((admin) => admin.command({ configureFailPoint: 'failCommand', mode: 'off' })); + return count; + }; +} + diff --git a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js new file mode 100644 index 0000000000..49f39f07d8 --- /dev/null +++ b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js @@ -0,0 +1,101 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { + waitForDaemonReady, waitForNodeStatus, waitFor, +} from '../framework/wait.js'; +import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; +import { failWrites } from '../framework/mongo-control.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node joining the network asks a peer for the network's app state, and the peer +// answers in several messages, only the last saying it is done. Here the joining +// node's database fails the write of the first message, and the node must not +// count that peer as having answered: it ends the request and asks another peer. +// +// The harness keeps one sync request open at a time (appSyncMinCompletions), so a +// peer counted despite the failed write leaves one peer asked, and a peer replaced +// leaves two. syncAnswerBatchSize 2 splits an answer of a few running-app events +// into several messages, and a long locationTtlS keeps the fleet from announcing +// again while the node syncs, so the only running-app writes it makes then are the +// answer's. + +const ANSWER_BATCH = 2; +const NODES = 6; +const JOINING = NODES - 1; +// what the peers hold when the node joins: more events than one message carries +const ANSWERED_EVENTS = ANSWER_BATCH + 1; +// a peer's request ended for any of these reasons is replaced too, and is not this +const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified', 'sync:refused']; + +function step(what) { + console.log(`# 3304 ${new Date().toISOString()} ${what}`); +} + +describe('a sync answer the node cannot record is not counted', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2esyncwrite${Date.now()}`; + let release; + let joining; + let eventsHeld; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + deferredNodes: 1, + tickerAutostart: false, + configOverrides: { fluxapps: { syncAnswerBatchSize: ANSWER_BATCH, locationTtlS: 1800 } }, + }); + await bootAndPeer(env); + await seedSimpleApp(env, appName); + await waitFor(async () => (await dbClient(1).appStateEvents({ type: 'apprunning' })).length >= ANSWERED_EVENTS, { + timeout: 180000, interval: 2000, label: 'the peers hold more running-app events than one message carries', + }); + eventsHeld = (await dbClient(1).appStateEvents({ type: 'apprunning' })).length; + + step('arming the joining node\'s first location write to fail'); + release = await failWrites(JOINING + 1, 'globalzelapps', 'zelappslocation', { times: 1 }); + joining = await env.startNode(JOINING); + await waitForDaemonReady(joining); + await waitForNodeStatus(joining, (d) => d.confirmed === true, 60000); + await joining.waitForEvent('ephemeralSync:allComplete', () => true, 180000); + step('synced'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('was answered in more than one message', function () { + expect(eventsHeld, 'the answer fits one message, so no early message could fail').to.be.above(ANSWER_BATCH); + }); + + it('had one of its location writes refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database failed none of the joining node\'s writes, so nothing below was tested').to.equal(1); + }); + + it('asks another peer in place of the one whose answer it could not record', function () { + const asked = new Set(joining.getEventBuffer() + .filter((e) => e.event === 'ephemeralSync:requested') + .flatMap((e) => e.data.peers)); + const otherEndings = joining.getEventBuffer().filter((e) => OTHER_ENDINGS.includes(e.event)); + expect(otherEndings, 'a request ended for another reason, so a second peer proves nothing').to.deep.equal([]); + expect(asked.size, 'the peer whose answer was not recorded was counted, and nobody else was asked').to.equal(2); + }); + + it('names that peer, and never counts it', function () { + const events = joining.getEventBuffer(); + const failedAt = events.find((e) => e.event === 'ephemeralSync:peerWriteFailed'); + expect(failedAt, 'the node said nothing about the answer it could not record').to.not.equal(undefined); + const counted = events.filter((e) => e.event === 'ephemeralSync:peerComplete' && e.data.peer === failedAt.data.peer); + expect(counted, 'the peer whose answer was not recorded was counted').to.deep.equal([]); + }); +}); From 85ea5adbe04ac8af3854547a98da125af05cd9c2 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:47:39 +0100 Subject: [PATCH 108/369] test(harness): 3304 runs on the production sync budget and asks only about the running-app answer The per-peer deadlines derive from syncTimeoutMs, and an answer cut into messages of two outlasts the harness's. The node's own refusals of other peers' requests are not endings of its request, and a stream a peer finished before its failed write stands. Co-Authored-By: Claude Opus 5.5 --- ...r-the-node-cannot-record-is-not-counted.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js index 49f39f07d8..d8e7fa2983 100644 --- a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js +++ b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js @@ -27,7 +27,7 @@ const JOINING = NODES - 1; // what the peers hold when the node joins: more events than one message carries const ANSWERED_EVENTS = ANSWER_BATCH + 1; // a peer's request ended for any of these reasons is replaced too, and is not this -const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified', 'sync:refused']; +const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified']; function step(what) { console.log(`# 3304 ${new Date().toISOString()} ${what}`); @@ -48,7 +48,16 @@ describe('a sync answer the node cannot record is not counted', function () { nodes: NODES, deferredNodes: 1, tickerAutostart: false, - configOverrides: { fluxapps: { syncAnswerBatchSize: ANSWER_BATCH, locationTtlS: 1800 } }, + configOverrides: { + fluxapps: { + syncAnswerBatchSize: ANSWER_BATCH, + locationTtlS: 1800, + // the production budget: the per-peer deadlines derive from it, and at + // the harness's 30s the wait between two messages of an answer cut this + // small is shorter than the peer takes to send them + syncTimeoutMs: 120000, + }, + }, }); await bootAndPeer(env); await seedSimpleApp(env, appName); @@ -91,11 +100,13 @@ describe('a sync answer the node cannot record is not counted', function () { expect(asked.size, 'the peer whose answer was not recorded was counted, and nobody else was asked').to.equal(2); }); - it('names that peer, and never counts it', function () { + it('names that peer, and never counts its running-app answer', function () { const events = joining.getEventBuffer(); const failedAt = events.find((e) => e.event === 'ephemeralSync:peerWriteFailed'); expect(failedAt, 'the node said nothing about the answer it could not record').to.not.equal(undefined); - const counted = events.filter((e) => e.event === 'ephemeralSync:peerComplete' && e.data.peer === failedAt.data.peer); + // the streams it finished before the failed write stand; the one it could not record does not + const counted = events.filter((e) => e.event === 'ephemeralSync:peerComplete' + && e.data.syncType === 'apprunning' && e.data.peer === failedAt.data.peer); expect(counted, 'the peer whose answer was not recorded was counted').to.deep.equal([]); }); }); From 8a3a53a5919905a48e54c34c5f1046448666c7b5 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 10:50:06 +0100 Subject: [PATCH 109/369] fix(appsync): a peer is credited only while its sync request is open A message admitted while the request was open can finish storing after the request ended, by its deadline or for an answer that could not be verified or recorded. Its completion is not counted. Co-Authored-By: Claude Opus 5.5 --- .../appMessaging/appSyncOrchestrator.js | 6 ++++++ tests/unit/appSyncOrchestrator.test.js | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index 1c06ff1c06..77b96ef308 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -315,6 +315,12 @@ class AppSyncOrchestrator { log.error(`AppSyncOrchestrator - ${syncType} sync complete with no peer, not counted`); return; } + // A message admitted while the request was open can finish after it ended: + // timed out, or ended for an answer that could not be verified or recorded. + if (!this.#requests.isOpen(peerKey)) { + log.info(`AppSyncOrchestrator - ${syncType} sync complete from ${peerKey} after its request ended, not counted`); + return; + } answered.add(peerKey); // Its answer is in, so it is no longer something being waited on. The // record stands as answered rather than being dropped: a peer that has diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index 1a54d47335..2731045472 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -816,6 +816,26 @@ describe('AppSyncOrchestrator', () => { return orchestrator; }; + it('does not credit a peer whose request ended while its last message was being stored', async () => { + const orchestrator = await driveToRequests(makeEligiblePeers(3)); + appSyncEvents.emit(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, makeEligiblePeers(3)[0].key); + await clock.tickAsync(0); + + completeAllTypes(3, ['apprunning', 'appinstalling', 'apperrors', 'apptemp']); + await clock.tickAsync(0); + + expect(orchestrator.state, 'a peer was credited after its request had ended').to.equal(STATES.SYNCING); + }); + + it('a canary: the same completions with every request open complete the sync', async () => { + const orchestrator = await driveToRequests(makeEligiblePeers(3)); + + completeAllTypes(3, ['apprunning', 'appinstalling', 'apperrors', 'apptemp']); + await clock.tickAsync(0); + + expect(orchestrator.state).to.equal(STATES.READY); + }); + it('does not complete the sync on the three surveys alone', async () => { const orchestrator = await driveToRequests(makeEligiblePeers(3)); From f682dd46b8ac2adb4ac36c7073902d2daf9f5c0c Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:13:01 +0100 Subject: [PATCH 110/369] fix(appsync): a hash retry writes the registry rows of the apps whose messages it stores On a node whose registry is built, a bulk-fetched message's app row is written before its hash is marked held; a row not written leaves the hash unmarked, and the next retry writes it from the stored message. A registry not built yet is built in full on the way to ready, as before. A retry no longer rebuilds the whole registry. Co-Authored-By: Claude Opus 5.5 --- .../appMessaging/appHashSyncService.js | 19 +++++- .../appMessaging/appSyncOrchestrator.js | 7 --- tests/unit/appHashSyncService.test.js | 62 +++++++++++++++++++ tests/unit/appSyncOrchestrator.test.js | 12 ++-- 4 files changed, 86 insertions(+), 14 deletions(-) diff --git a/ZelBack/src/services/appMessaging/appHashSyncService.js b/ZelBack/src/services/appMessaging/appHashSyncService.js index 1fda027fc4..d38556cdb5 100644 --- a/ZelBack/src/services/appMessaging/appHashSyncService.js +++ b/ZelBack/src/services/appMessaging/appHashSyncService.js @@ -21,6 +21,7 @@ const { appSyncEvents, EVENTS } = require('../utils/appSyncEvents'); const { HASH_EXPIRY_BLOCKS, HASH_RETRY_BACKOFF } = require('../utils/appConstants'); const log = require('../../lib/log'); const fluxEventBus = require('../utils/fluxEventBus'); +const globalState = require('../utils/globalState'); const { isBefore, messagesThatCount } = require('../utils/appMessageChain'); const { invalidMessages } = require('../invalidMessages'); const { Privilege, authOf } = require('../utils/privileges'); @@ -465,8 +466,9 @@ async function processMessages(messages, records, onProgress) { } // 5. Batch insert permanent messages (ordered so insertedCount is deterministic on failure) + let inserted = 0; if (permInserts.length > 0) { - let inserted = permInserts.length; + inserted = permInserts.length; await appsGlobalDb.collection(globalAppsMessages).insertMany(permInserts, { ordered: true }) .catch((err) => { inserted = err.result?.insertedCount ?? 0; @@ -476,6 +478,21 @@ async function processMessages(messages, records, onProgress) { processed += inserted; } + // The registry rows of the apps whose messages were stored, before their hashes are + // marked. A registry not built yet is built in full from every stored message before + // the node is ready (dbReady), so only a built one is kept current here. Rows not + // written leave the hashes unmarked, and the next retry finds the messages stored and + // writes the rows (alignStoredMessagesWithPayments). + if (inserted > 0 && globalState.dbReady) { + const apps = [...new Set(permInserts.slice(0, inserted).map((message) => message.appSpecifications.name))]; + try { + await messageVerifier.storeGoverningAppSpecifications(apps); + } catch (err) { + log.error(`processMessages registry rows: ${err.message}`); + hashMarkOps = []; + } + } + // 6. Batch mark hashes if (hashMarkOps.length > 0) { await daemonDb.collection(appsHashesCollection).bulkWrite(hashMarkOps, { ordered: false }) diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index 77b96ef308..a65162af4c 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -915,13 +915,6 @@ class AppSyncOrchestrator { } const result = await appHashSyncService.syncMissingHashes({ currentHeight: this.#lastBlockHeight }); this.#nextHashRetryHeight = result.nextRetryHeight ?? (this.#lastBlockHeight + FALLBACK_RECHECK_BLOCKS); - // A retry stores the messages it resolves without promoting them, so nothing else applies - // them: a renewal fetched here would never reach globalAppsInformation, and the app would - // expire on this node alone. Rebuild from the log, as the initial sync does. - if (result.resolved > 0) { - await registryManager.reindexGlobalAppsInformation() - .catch((error) => log.error(`AppSyncOrchestrator - Rebuild after hash retry failed: ${error.message}`)); - } if (result.missing > 0) { log.info(`AppSyncOrchestrator - Hash retry: ${result.resolved} resolved, ${result.missing} remaining, next check at block ${this.#nextHashRetryHeight}`); } diff --git a/tests/unit/appHashSyncService.test.js b/tests/unit/appHashSyncService.test.js index 386b68ce92..31aee9d48f 100644 --- a/tests/unit/appHashSyncService.test.js +++ b/tests/unit/appHashSyncService.test.js @@ -878,8 +878,10 @@ describe('appHashSyncService tests', () => { let localMessageVerifierStub; let localCheckAndDecryptStub; let localLogStub; + let localGlobalState; beforeEach(() => { + localGlobalState = { dbReady: true }; localCollectionStub = { bulkWrite: sinon.stub().resolves({ ok: 1 }), insertMany: sinon.stub().resolves({ insertedCount: 0 }), @@ -902,6 +904,7 @@ describe('appHashSyncService tests', () => { localMessageVerifierStub = { alignStoredMessagesWithPayments: sinon.stub().resolves([]), checkAndRequestApp: sinon.stub().resolves(true), + storeGoverningAppSpecifications: sinon.stub().resolves(), appHashHasMessage: sinon.stub().resolves(), appHashHasMessageNotFound: sinon.stub().resolves(), checkAndRequestMultipleApps: sinon.stub().resolves(), @@ -940,6 +943,65 @@ describe('appHashSyncService tests', () => { '../fluxCommunicationUtils': { deterministicFluxList: sinon.stub().resolves([]) }, '../fluxCommunication': { openEphemeralConnection: sinon.stub().resolves(null) }, '../fluxNetworkHelper': { getLocalSocketAddress: sinon.stub().resolves('10.0.0.99:16127') }, + '../utils/globalState': localGlobalState, + }); + }); + + describe('the registry rows of the apps it stores', () => { + const streamOne = () => { + const bulkMessages = [{ + type: 'fluxappregister', version: 1, hash: 'hash5', timestamp: Date.now(), + signature: 'sig5', appSpecifications: { name: 'lateapp', version: 4, owner: 'owner' }, + valueSat: 105, txid: 'tx5', height: 1005, + }]; + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1000 + i, value: 100 + i, message: false, + })); + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + return Promise.resolve(getMissingCalls === 1 ? manyMissing : []); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse(bulkMessages)); + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + }; + // the write that marks a hash's message as held + const hashMarks = () => localCollectionStub.bulkWrite.getCalls() + .filter((call) => call.args[0].some((op) => op.updateOne?.update?.$set?.message === true)); + + it('writes them on a node whose registry is built, before it marks their hashes', async () => { + streamOne(); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.called, 'the fixture stored nothing').to.be.true; + sinon.assert.calledOnceWithExactly(localMessageVerifierStub.storeGoverningAppSpecifications, ['lateapp']); + expect(hashMarks(), 'the hash was never marked').to.have.length(1); + expect(localMessageVerifierStub.storeGoverningAppSpecifications.calledBefore(hashMarks()[0])).to.equal(true); + }); + + it('leaves them to the full rebuild on a node whose registry is not built yet', async () => { + localGlobalState.dbReady = false; + streamOne(); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.called, 'the fixture stored nothing').to.be.true; + sinon.assert.notCalled(localMessageVerifierStub.storeGoverningAppSpecifications); + expect(hashMarks()).to.have.length(1); + }); + + it('marks no hash whose app\'s row could not be written, so the next retry writes it', async () => { + localMessageVerifierStub.storeGoverningAppSpecifications.rejects(new Error('connection timed out')); + streamOne(); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.called, 'the fixture stored nothing').to.be.true; + expect(hashMarks()).to.deep.equal([]); }); }); diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index 2731045472..03aadb04dd 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -2193,7 +2193,9 @@ describe('AppSyncOrchestrator', () => { expect(syncMissingHashesStub.calledTwice).to.be.true; }); - it('should rebuild globalAppsInformation when a retry resolves messages, and only then', async () => { + // A retry's messages reach the registry as they are stored (appHashSyncService), so + // the registry is rebuilt in full only on the way to ready. + it('does not rebuild globalAppsInformation after a retry, whatever it resolved', async () => { syncMissingHashesStub.onFirstCall().resolves({ resolved: 0, missing: 2, unreachable: 0, nextRetryHeight: 2555200 }); syncMissingHashesStub.onSecondCall().resolves({ resolved: 0, missing: 2, unreachable: 0, nextRetryHeight: 2555300 }); syncMissingHashesStub.onThirdCall().resolves({ resolved: 2, missing: 0, unreachable: 0, nextRetryHeight: null }); @@ -2205,15 +2207,13 @@ describe('AppSyncOrchestrator', () => { await clock.tickAsync(0); const afterInitialSync = reindexStub.callCount; - // a retry that resolves nothing has nothing to apply blockEmitter.emit('blocksProcessed', 2555200); await clock.tickAsync(0); - expect(reindexStub.callCount).to.equal(afterInitialSync); - - // the messages a retry resolves are stored without promotion: they reach the specs by a rebuild blockEmitter.emit('blocksProcessed', 2555300); await clock.tickAsync(0); - expect(reindexStub.callCount).to.equal(afterInitialSync + 1); + + expect(syncMissingHashesStub.calledThrice, 'the retry that resolved messages never ran').to.equal(true); + expect(reindexStub.callCount).to.equal(afterInitialSync); }); it('should use fallback interval when no hashes are backed off', async () => { From 5f9b2fe8422b12be1df85c71dedcfbe868623a3f Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:32:27 +0100 Subject: [PATCH 111/369] test(appsync): a partly stored bulk insert writes the rows of only the apps it stored Co-Authored-By: Claude Opus 5.5 --- tests/unit/appHashSyncService.test.js | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/unit/appHashSyncService.test.js b/tests/unit/appHashSyncService.test.js index 31aee9d48f..3ebfa5e322 100644 --- a/tests/unit/appHashSyncService.test.js +++ b/tests/unit/appHashSyncService.test.js @@ -994,6 +994,35 @@ describe('appHashSyncService tests', () => { expect(hashMarks()).to.have.length(1); }); + it('writes the rows of only the apps whose messages the insert stored', async () => { + const bulkMessages = ['firstapp', 'secondapp'].map((name, i) => ({ + type: 'fluxappregister', version: 1, hash: `hash${5 + i}`, timestamp: Date.now(), + signature: `sig${5 + i}`, appSpecifications: { name, version: 4, owner: 'owner' }, + valueSat: 105 + i, txid: `tx${5 + i}`, height: 1005 + i, + })); + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1000 + i, value: 100 + i, message: false, + })); + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + return Promise.resolve(getMissingCalls === 1 ? manyMissing : []); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse(bulkMessages)); + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + const partial = new Error('duplicate key'); + partial.result = { insertedCount: 1 }; + localCollectionStub.insertMany.rejects(partial); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.firstCall.args[0], 'the fixture inserted fewer than two').to.have.length(2); + sinon.assert.calledOnceWithExactly(localMessageVerifierStub.storeGoverningAppSpecifications, ['firstapp']); + }); + it('marks no hash whose app\'s row could not be written, so the next retry writes it', async () => { localMessageVerifierStub.storeGoverningAppSpecifications.rejects(new Error('connection timed out')); streamOne(); From a2ea5a8a6388b8a5acf1874f5962960cd14c9936 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:32:27 +0100 Subject: [PATCH 112/369] test(harness): 3305, a running node applies late messages app by app A running node more than 500 messages behind takes them in bulk, holds every app they bring at its newest message, and rewrites no other row: an app it already held keeps its row, and so does one the late log does not touch. Co-Authored-By: Claude Opus 5.5 --- ...g-node-applies-late-messages-app-by-app.js | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js diff --git a/test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js b/test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js new file mode 100644 index 0000000000..cd3f929553 --- /dev/null +++ b/test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js @@ -0,0 +1,146 @@ +/* + * A node that is running finds more than 500 app messages missing behind payments its scan + * recorded, and its hash retry fetches them in bulk. It writes the registry row of each app + * those messages belong to, and touches no other: the registry is built in full only on the + * way to ready. + * + * The node boots holding two apps. The late log then reaches the peers, and the node's + * payment records for it are written as its scan of the chain would have written them: + * ten new apps with long update chains, and late updates to one of the two it already holds. + * The other is untouched by the late log, so its row is only rewritten by a full rebuild, + * which replaces every row of the registry. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { waitForDaemonReady, waitForOrchestratorState, waitFor } from '../framework/wait.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const NODES = 4; +// the running node: held back until the apps it boots holding are in place +const SUBJECT = NODES - 1; +const LATE_APPS = 10; +const UPDATES = 54; +const LATE_UPDATES_TO_HELD = 5; + +const unresolved = (entry) => ({ ...entry, message: false }); + +function step(what) { + console.log(`# 3305 ${new Date().toISOString()} ${what}`); +} + +describe('a running node applies late messages app by app', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + // the newest message of each app the late log brings, by name + const governing = new Map(); + let untouched; + let updated; + let rowsBefore; + let lateCount; + let bulkFetched; + const db = () => dbClient(SUBJECT + 1); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + deferredNodes: 1, + // the retry that finds the late log comes within a few blocks + configOverrides: { fluxapps: { hashSyncFallbackRecheckBlocks: 2 } }, + }); + await bootAndPeer(env); + const base = env.initialHeight - 2000; + + // the two apps the node boots holding, its rows for them written by its own rebuild + untouched = await buildSeedableApp({ env, name: `e2eHeld${stamp}`, height: base }); + updated = await buildSeedableApp({ env, name: `e2eHeldUp${stamp}`, height: base + 1 }); + const held = [untouched, updated]; + await Promise.all(Array.from({ length: NODES }, (_, i) => i).map(async (i) => { + await dbClient(i + 1).seedPermanentMessages(held.map((m) => m.permanentMessage)); + await dbClient(i + 1).seedAppHashes(held.map((m) => m.hashEntry)); + })); + const subject = await env.startNode(SUBJECT); + await waitForDaemonReady(subject); + await env.startDiscovery([SUBJECT]); + await waitForOrchestratorState(subject, 'READY', 600000); + await waitFor(async () => (await db().globalAppSpec(updated.permanentMessage.appSpecifications.name))?.hash === updated.hash + && (await db().globalAppSpec(untouched.permanentMessage.appSpecifications.name))?.hash === untouched.hash, { + timeout: 120000, interval: 2000, label: 'the node holds the two apps it booted with', + }); + rowsBefore = { + untouched: await db().globalAppSpec(untouched.permanentMessage.appSpecifications.name), + updated: await db().globalAppSpec(updated.permanentMessage.appSpecifications.name), + }; + + // the late log + const late = []; + let current = updated; + for (let j = 1; j <= LATE_UPDATES_TO_HELD; j += 1) { + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableUpdate(current, (spec) => { spec.description = `held app revision ${j}`; }, { height: base + 2 + j }); + late.push(current); + } + governing.set(updated.permanentMessage.appSpecifications.name, current.hash); + for (let k = 0; k < LATE_APPS; k += 1) { + const name = `e2eLate${k}x${stamp}`; + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableApp({ env, name, height: base + 100 + (k * 60) }); + late.push(current); + for (let j = 1; j <= UPDATES; j += 1) { + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableUpdate(current, (spec) => { spec.description = `${name} revision ${j}`; }, { height: base + 100 + (k * 60) + j }); + late.push(current); + } + governing.set(name, current.hash); + } + lateCount = late.length; + expect(lateCount, 'enough messages late to take the bulk path').to.be.above(500); + + const anchor = subject.getLastEventId(); + step(`seeding ${lateCount} late messages`); + await Promise.all(env.clients.map(async (client, i) => { + if (!client || i === SUBJECT) return; + await dbClient(i + 1).seedPermanentMessages(late.map((m) => m.permanentMessage)); + await dbClient(i + 1).seedAppHashes(late.map((m) => m.hashEntry)); + })); + await db().seedAppHashes(late.map((m) => unresolved(m.hashEntry))); + bulkFetched = await subject.waitForEvent('hashSync:bulkFetched', (d) => d.processed > 0, 600000, { afterId: anchor }); + step('fetched in bulk'); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('took the late messages in bulk while running', function () { + expect(bulkFetched.data.processed, JSON.stringify(bulkFetched.data)).to.equal(lateCount); + }); + + it('holds every app the late log brings at its newest message', async function () { + this.timeout(180000); + const names = [...governing.keys()]; + await waitFor(async () => { + const held = await Promise.all(names.map(async (name) => (await db().globalAppSpec(name))?.hash)); + return held.every((hash, i) => hash === governing.get(names[i])); + }, { timeout: 150000, interval: 3000, label: 'every late app held at its newest message' }); + }); + + it('brings an app it already held up to its late update, in the same row', async function () { + const row = await db().globalAppSpec(updated.permanentMessage.appSpecifications.name); + expect(row.hash).to.equal(governing.get(updated.permanentMessage.appSpecifications.name)); + expect(String(row._id), 'the row was replaced, so the registry was rebuilt').to.equal(String(rowsBefore.updated._id)); + }); + + it('rewrites no row the late log does not touch', async function () { + const row = await db().globalAppSpec(untouched.permanentMessage.appSpecifications.name); + expect(row, 'the app the late log does not touch is gone').to.not.equal(null); + expect(String(row._id), 'the row was replaced, so the registry was rebuilt').to.equal(String(rowsBefore.untouched._id)); + }); +}); From 8d853f22323226d3e54a5bd4fe30436f608f44c5 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:37:55 +0100 Subject: [PATCH 113/369] fix(apps): a composed app's redeploy replaces its row with the new specification in one write The row is never deleted: until the write lands it holds the old specification, so an app whose write fails still has a row describing it, which the reconciler rebuilds from and the next reinstall pass finds again. The requirements are checked before the write. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 21 +++++-------------- tests/unit/advancedWorkflows.test.js | 21 +++++++++++++++---- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index cdfac378e6..ce59af196c 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -4617,21 +4617,13 @@ async function reinstallOldApplications() { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); const appsQuery = { name: appSpecifications.name }; - const appsProjection = {}; - log.warn('Cleaning up database...'); - // eslint-disable-next-line no-await-in-loop - await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - const databaseStatus2 = { - status: 'Database cleaned', - }; - log.warn('Database cleaned'); - log.warn(databaseStatus2); log.warn(`Compositions of application ${appSpecifications.name} uninstalled. Continuing with installation...`); - // composition removal done. Remove from installed apps and being installation // eslint-disable-next-line no-await-in-loop await appInstaller.checkAppRequirements(appSpecifications); // entire app - // Register the app in database BEFORE creating Docker containers to prevent race condition + // The app's row becomes the new specification in one write, before any + // container is created: until it lands the row still holds the old one, + // so the app always has a row describing it. const isEnterprise = Boolean( appSpecifications.version >= 8 && appSpecifications.enterprise, ); @@ -4644,11 +4636,8 @@ async function reinstallOldApplications() { } // eslint-disable-next-line no-await-in-loop - const insertResult = await dbHelper.insertOneToDatabase(appsDatabase, localAppsInformation, dbSpecs); - if (!insertResult) { - throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name} during composed app redeployment. Database insert returned undefined - likely duplicate key error or database failure. Aborting redeployment to prevent orphaned Docker containers.`); - } - log.info(`Database entry created for ${appSpecifications.name} BEFORE component Docker container creation (composed redeployment path)`); + await dbHelper.replaceOneInDatabase(appsDatabase, localAppsInformation, appsQuery, dbSpecs, { upsert: true }); + log.info(`Database entry of ${appSpecifications.name} replaced with its new specification BEFORE component Docker container creation (composed redeployment path)`); let allComponentsBack = true; const replaced = []; diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 4f4fbef661..e1c7f8fd83 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -7381,18 +7381,32 @@ describe('advancedWorkflows tests', () => { it('does not uninstall an app whose redeploy stops on a database that does not answer', async () => { sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); await advancedWorkflows.reinstallOldApplications(); expect(removeAppLocally.called, 'an app was uninstalled with its data because the database did not answer').to.be.false; }); + it('never deletes the app\'s row: the new specification replaces it in one write', async () => { + sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + const deleteRow = sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({}); + const replaceRow = sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + + await advancedWorkflows.reinstallOldApplications(); + + expect(replaceRow.called, 'the fixture never reached the row write').to.be.true; + expect(deleteRow.called, 'the row was deleted, so a failed write leaves the app with none').to.be.false; + }); + it('a canary: a redeploy that fails for another reason still uninstalls the app', async () => { sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').rejects(new Error('Insufficient RAM')); + sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new Error('Insufficient RAM')); await advancedWorkflows.reinstallOldApplications(); @@ -7602,8 +7616,7 @@ describe('advancedWorkflows tests', () => { beforeEach(() => { sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); sinon.stub(appUninstaller, 'removeAppLocally').resolves(); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({}); - sinon.stub(dbHelper, 'insertOneToDatabase').resolves({ acknowledged: true }); + sinon.stub(dbHelper, 'replaceOneInDatabase').resolves({ acknowledged: true }); sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); installApplicationSoft = sinon.stub(appInstaller, 'installApplicationSoft').resolves(); sinon.stub(appNetworkLinker, 'checkAppNetworkRequirements').resolves(); From 116d51afccb3168fd51a5455fab6ba3f5b486376 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:38:43 +0100 Subject: [PATCH 114/369] test(harness): 3306, a redeploy whose record write fails keeps the app The database refuses the write of the app's new record during the reinstall pass; the app keeps its record, and comes back at the new specification on a later pass. Co-Authored-By: Claude Opus 5.5 --- ...-whose-record-write-fails-keeps-the-app.js | 120 ++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js diff --git a/test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js b/test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js new file mode 100644 index 0000000000..88910c359d --- /dev/null +++ b/test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js @@ -0,0 +1,120 @@ +// weight: heavy +/* + * The periodic reinstall pass redeploys an app whose specification changed, and the + * database refuses the write of the app's new record. The app keeps its record, at the + * old specification, so the node still knows it holds the app: the reconciler restores + * what the pass took down, and a later pass redeploys it at the new specification. + * + * The change arrives as in suite 85: the app's global row is replaced and a new + * permanent message and hash seeded beside it. The pass fires on + * `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0`, every 16 to 36 + * blocks here, so the drive is budgeted in blocks. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { getAppContainerId } from '../framework/container.js'; +import { waitFor, waitForComponentRedeployed } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { failWrites } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// four passes at the longest period: the one whose write is refused, and others after it +const PASS_BLOCKS = 144; +const HOLDER = 0; + +describe('a redeploy whose record write fails keeps the app', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eredeployrow${Date.now()}`; + let holder; + let app; + let updated; + let containerBefore; + let release; + let beforeUpdate; + + async function localSpec(client) { + const res = await client.getInstalledApps(); + if (res.status !== 'success') return null; + return (res.data ?? []).find((a) => a.name === appName) ?? null; + } + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ hookCtx: this, nodes: 5, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await pushImage(appName, 'v1'); + app = await buildSeedableApp({ + env, + name: appName, + instances: 1, + compose: [{ + name: appName, + description: 'redeploy row test container', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + await installOnNodes(env, app, [HOLDER]); + await waitFor(async () => { + containerBefore = await getAppContainerId(holder.container, appName, appName); + return Boolean(containerBefore); + }, { timeout: 120000, interval: 2000, label: 'the app is running before its spec changes' }); + + // a description change takes the soft path, which keeps the data volume + updated = await buildSeedableUpdate(app, (spec) => { + spec.compose[0].description = 'redeploy row test container, second specification'; + }); + beforeUpdate = holder.getLastEventId(); + await seedSpecUpdate(env, updated, [HOLDER]); + // the only thing advancing the chain from here is the drive below + await stopTicker(); + release = await failWrites(HOLDER + 1, 'localzelapps', 'zelappsinformation', { times: 1 }); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('comes back at the new specification', async function () { + // a backstop: the drive is bounded in blocks, and blocks cost what the node takes + this.timeout(2400000); + const replaced = () => waitForComponentRedeployed(holder, appName, appName, false, 0, { afterId: beforeUpdate }) + .then(() => true, () => false); + await driveUntil(holder, replaced, { + blocks: PASS_BLOCKS, label: 'the node redeploys the app at the new specification', + }); + + const installed = await localSpec(holder); + expect(installed, 'the node no longer holds the app').to.not.equal(null); + expect(installed.hash, 'the row is at the new specification').to.equal(updated.hash); + const containerAfter = await getAppContainerId(holder.container, appName, appName); + expect(containerAfter, 'the app is not running').to.be.a('string'); + expect(containerAfter, 'the component was never replaced').to.not.equal(containerBefore); + }); + + it('had the write of the app\'s record refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database refused none of the node\'s record writes, so nothing above was tested').to.equal(1); + }); +}); From 66baa00325cb7d6478ce26a068d9cfa5e22366b0 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:56:12 +0100 Subject: [PATCH 115/369] refactor(apps): telling the network a node no longer holds an app is one function appUninstaller.announceRemoval broadcasts fluxappremoved and drops the app from the running apps cache; removeAppLocally calls it when its removal is announced. Co-Authored-By: Claude Opus 5.5 --- .../services/appLifecycle/appUninstaller.js | 68 ++++++++++--------- 1 file changed, 37 insertions(+), 31 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 8fd8b8902c..ba22d3735a 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -57,6 +57,41 @@ function setOnComponentRemoved(callback) { onComponentRemoved = callback; } +/** + * Tells the network this node no longer holds the app (fluxappremoved), and drops it + * from the running apps cache. For an app whose removal from this node is complete. + * @param {string} appName + * @returns {Promise} + */ +async function announceRemoval(appName) { + // An announcement cycle that took its list before this removal marked the + // app still names it, and a claim arriving after this message re-creates the + // row it cleared. Waiting for the cycle in flight puts the two on the wire in + // the order they happened, so the claim is applied first and this clears it. + // A cycle starting from here on reads the mark and leaves the app out. + // Bounded: a wedged cycle must not hold up the node's removals. + await globalState.announceCycle.readyTimeout(ANNOUNCE_CYCLE_WAIT_MS); + const ip = await fluxNetworkHelper.getLocalSocketAddress(); + if (!ip) { + log.warn(`${appName} removed without announcing it - this node's own address is unknown, so peers hold its location row until it expires`); + return; + } + const appRemovedMessage = { + type: 'fluxappremoved', + version: 1, + appName, + ip, + broadcastedAt: Date.now(), + }; + log.info('Broadcasting appremoved message to the network'); + await fluxCommunicationMessagesSender.broadcastMessageToAll(appRemovedMessage); + const { runningAppsCache } = globalState; + if (runningAppsCache.has(appName)) { + runningAppsCache.delete(appName); + log.info(`Removed ${appName} from running apps cache`); + } +} + /** * Stop Syncthing app and clean up cache * @param {string} monitoredName - Monitored app name @@ -982,37 +1017,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen fluxEventBus.publish('app:removed', { name: appName }); - if (sendMessage) { - // An announcement cycle that took its list before this removal marked the - // app still names it, and a claim arriving after this message re-creates the - // row it cleared. Waiting for the cycle in flight puts the two on the wire in - // the order they happened, so the claim is applied first and this clears it. - // A cycle starting from here on reads the mark and leaves the app out. - // Bounded: a wedged cycle must not hold up the node's removals. - await globalState.announceCycle.readyTimeout(ANNOUNCE_CYCLE_WAIT_MS); - const ip = await fluxNetworkHelper.getLocalSocketAddress(); - if (ip) { - const broadcastedAt = Date.now(); - const appRemovedMessage = { - type: 'fluxappremoved', - version: 1, - appName, - ip, - broadcastedAt, - }; - log.info('Broadcasting appremoved message to the network'); - // broadcast messages about app removed to all peers - await fluxCommunicationMessagesSender.broadcastMessageToAll(appRemovedMessage); - // Remove app from running apps cache - const { runningAppsCache } = globalState; - if (runningAppsCache.has(appName)) { - runningAppsCache.delete(appName); - log.info(`Removed ${appName} from running apps cache`); - } - } else { - log.warn(`${appName} removed without announcing it - this node's own address is unknown, so peers hold its location row until it expires`); - } - } + if (sendMessage) await announceRemoval(appName); if (!isComponent) { const dockerNetworkStatus = { @@ -1333,6 +1338,7 @@ async function removeAppLocallyApi(req, res) { } module.exports = { + announceRemoval, hardUninstallComponent, hardUninstallApplication, softUninstallComponent, From ee0381aecba0eca5ab0b735182f1515c59ba4c56 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:56:12 +0100 Subject: [PATCH 116/369] fix(apps): a hard redeploy is one function, and one that removed an app it could not reinstall tells the network appRebuild.rebuildApp removes an app and installs it again at a specification. A removal that does not finish leaves the app and its row; a finished removal followed by an install that does not complete leaves the node without the app, and the network is told once. hardRedeploy and the reinstall pass's upgrade of an app from the old specification format both rebuild through it, and a rebuild that does not complete no longer stops the reinstall pass. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 118 +++--------------- .../src/services/appLifecycle/appRebuild.js | 47 +++++++ tests/unit/advancedWorkflows.test.js | 33 +++++ tests/unit/appRebuild.test.js | 88 +++++++++++++ 4 files changed, 182 insertions(+), 104 deletions(-) create mode 100644 ZelBack/src/services/appLifecycle/appRebuild.js create mode 100644 tests/unit/appRebuild.test.js diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index ce59af196c..581b5ed6be 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -20,6 +20,7 @@ const { collateralOutpoint, nodesNameThisNode } = require('../utils/nodePinning' const fluxEventBus = require('../utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../utils/addressMoveEvents'); const { InstallOutcome } = require('../utils/installOutcome'); +const appRebuild = require('./appRebuild'); const generalService = require('../generalService'); const placementFeasibility = require('../appPlacement/placementFeasibility'); // eslint-disable-next-line no-unused-vars @@ -1682,11 +1683,6 @@ async function softRedeploy(appSpecs, res) { * @param {object} res - Response object */ async function hardRedeploy(appSpecs, res) { - // eslint-disable-next-line global-require - const appUninstaller = require('./appUninstaller'); - // Whether the teardown was reached. A failure before it leaves a running app - // running: the forced uninstall below answers only a failed rebuild. - let teardownStarted = false; try { // Every operation, including the periodic reinstall pass - which sets its // flag before it tears anything down and holds it across the wait, so the @@ -1706,20 +1702,7 @@ async function hardRedeploy(appSpecs, res) { globalState.hardRedeployInProgress = true; log.warn(`REMOVAL REASON: Hard redeploy initiated - ${appSpecs.name} being removed as part of hard redeploy process (hardRedeploy)`); - teardownStarted = true; - await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); - const appRedeployResponse = messageHelper.createSuccessMessage('Application removed. Awaiting installation...'); - log.info(appRedeployResponse); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - // verify requirements - // eslint-disable-next-line global-require - const appInstaller = require('./appInstaller'); - await appInstaller.checkAppRequirements(appSpecs); - // register - const outcome = await appInstaller.registerAppLocally(appSpecs, undefined, res, false, true); // can throw + const outcome = await appRebuild.rebuildApp(appSpecs, res); if (outcome !== InstallOutcome.INSTALLED) { const notReinstalled = messageHelper.createErrorMessage( `Application ${appSpecs.name} was not reinstalled (${outcome})`, @@ -1735,23 +1718,17 @@ async function hardRedeploy(appSpecs, res) { log.info('Application redeployed'); globalState.hardRedeployInProgress = false; } catch (error) { + // rebuildApp settles every failure of its own, so this is a failure before it ran log.error(error); globalState.hardRedeployInProgress = false; - if (!teardownStarted) { - const notStarted = messageHelper.createErrorMessage( - `Hard redeploy of ${appSpecs.name} failed before the app was taken down: ${error.message}`, - ); - log.warn(notStarted); - if (res) { - res.write(serviceHelper.ensureString(notStarted)); - if (res.flush) res.flush(); - } - return; + const notStarted = messageHelper.createErrorMessage( + `Hard redeploy of ${appSpecs.name} failed before the app was taken down: ${error.message}`, + ); + log.warn(notStarted); + if (res) { + res.write(serviceHelper.ensureString(notStarted)); + if (res.flush) res.flush(); } - log.warn(`REMOVAL REASON: Hard redeploy failure - ${appSpecs.name} failed during hard redeploy: ${error.message} (hardRedeploy)`); - // endResponse false: redeployAPI opened this response and closes it. - await appUninstaller.removeAppLocally(appSpecs.name, res, true, false, true); - log.info(`Cleanup completed for ${appSpecs.name} after hard redeploy failure`); } } @@ -4374,79 +4351,12 @@ async function reinstallOldApplications() { // eslint-disable-next-line no-continue continue; } - log.warn('Updating from old application version, doing hard redeploy...'); log.warn(`REMOVAL REASON: App version upgrade - ${appSpecifications.name} upgrading from v${installedApp.version} to v${appSpecifications.version}`); - // eslint-disable-next-line global-require - const appUninstaller = require('./appUninstaller'); - // eslint-disable-next-line global-require - const appInstaller = require('./appInstaller'); - // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appSpecifications.name, null, true, false); - // connect to mongodb - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const appsQuery = { name: appSpecifications.name }; - const appsProjection = {}; - log.warn('Cleaning up database...'); - // eslint-disable-next-line no-await-in-loop - await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - const databaseStatus2 = { - status: 'Database cleaned', - }; - log.warn('Database cleaned'); - log.warn(databaseStatus2); - log.warn(`Compositions of application ${appSpecifications.name} uninstalled. Continuing with installation...`); - // composition removal done. Remove from installed apps and being installation + // A version 3 app has no components and a version 4 one has, so nothing of the + // old one carries over: the app is rebuilt from nothing. // eslint-disable-next-line no-await-in-loop - await appInstaller.checkAppRequirements(appSpecifications); // entire app - - // Register the app in database BEFORE creating Docker containers to prevent race condition - const isEnterprise = Boolean( - appSpecifications.version >= 8 && appSpecifications.enterprise, - ); - - const dbSpecs = JSON.parse(JSON.stringify(appSpecifications)); - - if (isEnterprise) { - dbSpecs.compose = []; - dbSpecs.contacts = []; - } - - // eslint-disable-next-line no-await-in-loop - const insertResult = await dbHelper.insertOneToDatabase(appsDatabase, localAppsInformation, dbSpecs); - if (!insertResult) { - throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name} during version upgrade reinstallation. Database insert returned undefined - likely duplicate key error or database failure. Aborting reinstallation to prevent orphaned Docker containers.`); - } - log.info(`Database entry created for ${appSpecifications.name} BEFORE component Docker container creation (version upgrade path)`); - - // Now install components - containers will be created but app is already in DB - let allComponentsBack = true; - const replaced = []; - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecifications.compose) { - log.warn(`Continuing Hard Redeployment of component ${appComponent.name}_${appSpecifications.name}...`); - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); - // install the app - // eslint-disable-next-line no-await-in-loop - const outcome = await appInstaller.registerAppLocally(appSpecifications, appComponent); // component - if (outcome !== InstallOutcome.INSTALLED) { - log.error(`Component ${appComponent.name}_${appSpecifications.name} was not reinstalled (${outcome}). ${appSpecifications.name} is part-built; the app row describes every component, so the reconciler recreates what is missing.`); - allComponentsBack = false; - break; - } - replaced.push({ component: appComponent.name, hard: true }); - } - // Announced and restarted only if it is actually back. Saying so - // regardless is what turned a refused install into a node reporting - // an app it had taken apart. - if (allComponentsBack) { - log.warn(`Composed application ${appSpecifications.name} updated.`); - log.warn(`Restarting application ${appSpecifications.name}`); - // eslint-disable-next-line no-await-in-loop, no-use-before-define - await appDockerRestart(appSpecifications.name); - announceComponentsRedeployed(appSpecifications.name, replaced); - } + const outcome = await appRebuild.rebuildApp(appSpecifications); + log.warn(`Application ${appSpecifications.name} upgraded to v${appSpecifications.version}: ${outcome}`); } else if (appSpecifications.version <= 3) { if (appSpecifications.tiered) { const hddTier = `hdd${tier}`; diff --git a/ZelBack/src/services/appLifecycle/appRebuild.js b/ZelBack/src/services/appLifecycle/appRebuild.js new file mode 100644 index 0000000000..7bc4ff5947 --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appRebuild.js @@ -0,0 +1,47 @@ +/** + * A hard redeploy: an app removed from this node, data and all, and installed again + * at a specification, the whole app at once. The one way an app is rebuilt from + * nothing; the callers decide when, and hold the node while it runs. + */ +const log = require('../../lib/log'); +const appUninstaller = require('./appUninstaller'); +const appInstaller = require('./appInstaller'); +const { RemovalOutcome } = require('../utils/removalOutcome'); +const { InstallOutcome } = require('../utils/installOutcome'); + +/** + * Remove the app and install it again at appSpecs. + * + * A removal that does not finish leaves the app as it was, with its row, for the + * reconciler. Once the removal has finished this node holds nothing of the app, so + * an install that does not complete is told to the network, once, here. + * @param {object} appSpecs The specification to install. + * @param {object} [res] A response stream the caller holds. + * @returns {Promise} The InstallOutcome; FAILED when the removal did not finish. + */ +async function rebuildApp(appSpecs, res) { + const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); + if (removal !== RemovalOutcome.REMOVED && removal !== RemovalOutcome.NOT_INSTALLED) { + log.warn(`appRebuild - ${appSpecs.name} was not removed (${removal}), so it is not reinstalled`); + return InstallOutcome.FAILED; + } + + let outcome; + try { + await appInstaller.checkAppRequirements(appSpecs); + // the installer's own teardown says nothing: whether to tell the network is decided below + outcome = await appInstaller.registerAppLocally(appSpecs, undefined, res, false, false); + } catch (error) { + log.error(`appRebuild - ${appSpecs.name} was not reinstalled: ${error.message}`); + outcome = InstallOutcome.FAILED; + } + + if (outcome !== InstallOutcome.INSTALLED && removal === RemovalOutcome.REMOVED) { + log.warn(`appRebuild - ${appSpecs.name} was removed and not reinstalled (${outcome}); telling the network`); + await appUninstaller.announceRemoval(appSpecs.name) + .catch((error) => log.error(`appRebuild - the removal of ${appSpecs.name} was not announced: ${error.message}`)); + } + return outcome; +} + +module.exports = { rebuildApp }; diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index e1c7f8fd83..08f827bb1a 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -27,6 +27,7 @@ const { PeerComponent } = require('../../ZelBack/src/services/appMonitoring/peer const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../../ZelBack/src/services/utils/addressMoveEvents'); const appInstaller = require('../../ZelBack/src/services/appLifecycle/appInstaller'); +const appRebuild = require('../../ZelBack/src/services/appLifecycle/appRebuild'); const appNetworkLinker = require('../../ZelBack/src/services/appLifecycle/appNetworkLinker'); const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); const appInspector = require('../../ZelBack/src/services/appManagement/appInspector'); @@ -7403,6 +7404,38 @@ describe('advancedWorkflows tests', () => { expect(deleteRow.called, 'the row was deleted, so a failed write leaves the app with none').to.be.false; }); + describe('an app upgraded from the old format', () => { + const oldFormat = { + version: 3, name: 'myapp', hash: 'oldhash', repotag: 'nginx:1.0', hdd: 5, + }; + + it('is rebuilt from nothing at its new specification', async () => { + const rebuildApp = sinon.stub(appRebuild, 'rebuildApp').resolves('installed'); + dbHelper.findInDatabase.resolves([oldFormat]); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(rebuildApp); + expect(rebuildApp.firstCall.args[0]).to.include({ name: 'myapp', hash: 'newhash', version: 4 }); + }); + + it('lets the pass go on to the next app when its rebuild does not complete', async () => { + const rebuildApp = sinon.stub(appRebuild, 'rebuildApp').resolves('failed'); + const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + sinon.stub(dbHelper, 'replaceOneInDatabase').resolves({ acknowledged: true }); + const other = { ...installedApp, name: 'otherapp' }; + dbHelper.findInDatabase.resolves([oldFormat, other]); + dbHelper.findOneInDatabase.callsFake(async (db, collection, query) => (query.name === 'otherapp' + ? { ...newSpec, name: 'otherapp' } : newSpec)); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(rebuildApp); + expect(softUninstallComponent.called, 'the pass stopped at the app whose rebuild failed').to.equal(true); + }); + }); + it('a canary: a redeploy that fails for another reason still uninstalls the app', async () => { sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); diff --git a/tests/unit/appRebuild.test.js b/tests/unit/appRebuild.test.js new file mode 100644 index 0000000000..141467fdc8 --- /dev/null +++ b/tests/unit/appRebuild.test.js @@ -0,0 +1,88 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); +const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutcome'); +const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); + +describe('appRebuild tests', () => { + let uninstaller; + let installer; + let appRebuild; + const spec = { name: 'myapp', version: 8 }; + + beforeEach(() => { + uninstaller = { + removeAppLocally: sinon.stub().resolves(RemovalOutcome.REMOVED), + announceRemoval: sinon.stub().resolves(), + }; + installer = { + checkAppRequirements: sinon.stub().resolves(true), + registerAppLocally: sinon.stub().resolves(InstallOutcome.INSTALLED), + }; + appRebuild = proxyquire('../../ZelBack/src/services/appLifecycle/appRebuild', { + './appUninstaller': uninstaller, + './appInstaller': installer, + '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, + }); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('removes the app and installs it again, and tells the network nothing', async () => { + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.INSTALLED); + + sinon.assert.callOrder(uninstaller.removeAppLocally, installer.checkAppRequirements, installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('leaves the installer\'s teardown silent, so the network is told once, here', async () => { + await appRebuild.rebuildApp(spec); + + expect(installer.registerAppLocally.firstCall.args[4], 'the installer was asked to announce its own teardown').to.equal(false); + }); + + it('tells the network once when the app was removed and the install did not complete', async () => { + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + + sinon.assert.calledOnceWithExactly(uninstaller.announceRemoval, 'myapp'); + }); + + it('tells the network when the app was removed and its requirements are no longer met', async () => { + installer.checkAppRequirements.rejects(new Error('Insufficient RAM')); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + + sinon.assert.notCalled(installer.registerAppLocally); + sinon.assert.calledOnceWithExactly(uninstaller.announceRemoval, 'myapp'); + }); + + it('installs nothing and tells the network nothing when the removal did not finish', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + + sinon.assert.notCalled(installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('tells the network nothing about an app that was not installed here', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.NOT_INSTALLED); + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + + await appRebuild.rebuildApp(spec); + + sinon.assert.calledOnce(installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('settles a removal it could not announce, and still answers', async () => { + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + uninstaller.announceRemoval.rejects(new Error('no peers')); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + }); +}); From d0f8bf195a36a586f8a9e6229ddaa0ce87c3a8b1 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 11:58:00 +0100 Subject: [PATCH 117/369] test(harness): 3307, an app upgraded from the old format is rebuilt An app at specification version 3 updated to one with components is rebuilt at it. On a node where the database refuses the rebuilt app's row, the node tells the network it no longer holds the app and the reinstall pass goes on to the next one. Co-Authored-By: Claude Opus 5.5 --- ...upgraded-from-the-old-format-is-rebuilt.js | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100644 test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js diff --git a/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js b/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js new file mode 100644 index 0000000000..4fa16e6278 --- /dev/null +++ b/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js @@ -0,0 +1,163 @@ +// weight: heavy +/* + * An app installed at specification version 3 is updated to a version that has + * components. Nothing of the old app carries over, so the reinstall pass rebuilds it + * from nothing: removed, and installed again at the new specification. + * + * On one node the rebuild completes, and the app runs at its new specification. On + * another the database refuses the write of the new app's row, so the node no longer + * holds the app and tells the network so; the pass goes on to the next app. + * + * The change arrives as in suite 85. The pass fires on + * `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0`, every 16 to 36 + * blocks here, so the drive is budgeted in blocks. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableLegacyApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { execInContainer, getAppContainerId } from '../framework/container.js'; +import { waitFor, waitForComponentRedeployed } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { failWrites } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const PASS_BLOCKS = 144; +const FAILS = 0; +const UPGRADES = 1; + +function step(what) { + console.log(`# 3307 ${new Date().toISOString()} ${what}`); +} + +const component = (name, description) => ({ + name, + description, + repotag: `${REGISTRY_REPO_HOST}/${name}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', +}); + +// The specification an old-format app is updated to: the one a new app of its name +// would carry, signed by its owner as an update. +async function upgradeOf(env, legacy) { + const { name } = legacy.spec; + const target = await buildSeedableApp({ env, name, instances: 1, compose: [component(name, `${name} upgraded`)] }); + return buildSeedableUpdate(legacy, (spec) => { + Object.keys(spec).forEach((key) => { delete spec[key]; }); + const { hash, height, ...fields } = target.spec; + Object.assign(spec, fields); + }); +} + +// an app at specification version 3 has one container, named for the app alone +async function legacyContainerId(container, name) { + const { stdout } = await execInContainer(container, `docker inspect --format '{{.Id}}' flux${name} 2>/dev/null || echo ""`); + return stdout.trim() || null; +} + +async function localSpec(client, name) { + const res = await client.getInstalledApps(); + if (res.status !== 'success') return null; + return (res.data ?? []).find((a) => a.name === name) ?? null; +} + +describe('an app upgraded from the old format is rebuilt', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + const failing = `e2elegacyfail${stamp}`; + const other = `e2elegacynext${stamp}`; + const upgrading = `e2elegacyup${stamp}`; + let upgraded; + let release; + let anchors; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 5, tickerAutostart: false }); + await bootAndPeer(env); + + await Promise.all([failing, upgrading, other].map((name) => pushImage(name, 'v1'))); + const legacyFailing = await buildSeedableLegacyApp({ env, name: failing }); + const legacyUpgrading = await buildSeedableLegacyApp({ env, name: upgrading }); + const otherApp = await buildSeedableApp({ env, name: other, instances: 1, compose: [component(other, `${other} first`)] }); + // in this order: the reinstall pass reaches the app it cannot rebuild first + await installOnNodes(env, legacyFailing, [FAILS]); + await installOnNodes(env, otherApp, [FAILS]); + await installOnNodes(env, legacyUpgrading, [UPGRADES]); + await waitFor(async () => Boolean(await legacyContainerId(env.clients[FAILS].container, failing)) + && Boolean(await getAppContainerId(env.clients[FAILS].container, other, other)) + && Boolean(await legacyContainerId(env.clients[UPGRADES].container, upgrading)), { + timeout: 180000, interval: 2000, label: 'every app runs before its specification changes', + }); + + const failingUpgrade = await upgradeOf(env, legacyFailing); + upgraded = await upgradeOf(env, legacyUpgrading); + const otherUpdate = await buildSeedableUpdate(otherApp, (spec) => { spec.compose[0].description = `${other} second`; }); + anchors = env.clients.map((client) => client?.getLastEventId()); + await seedSpecUpdate(env, failingUpgrade, [FAILS]); + await seedSpecUpdate(env, otherUpdate, [FAILS]); + await seedSpecUpdate(env, upgraded, [UPGRADES]); + await stopTicker(); + release = await failWrites(FAILS + 1, 'localzelapps', 'zelappsinformation', { times: 1 }); + step('the reinstall pass is due'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('goes on to the next app after one it could not rebuild', async function () { + this.timeout(2400000); + const node = env.clients[FAILS]; + const redeployed = () => waitForComponentRedeployed(node, other, other, false, 0, { afterId: anchors[FAILS] }) + .then(() => true, () => false); + await driveUntil(node, redeployed, { blocks: PASS_BLOCKS, label: 'the pass redeploys the next app' }); + }); + + it('tells the network it no longer holds the app it could not rebuild', async function () { + this.timeout(120000); + const node = env.clients[FAILS]; + expect(await localSpec(node, failing), 'the node still holds the app').to.equal(null); + const peers = env.clients.filter((client, i) => client && i !== FAILS); + const heard = await Promise.any(peers.map((peer, i) => peer.waitForEvent( + 'network:appremoved', + (d) => d.name === failing && d.ip.startsWith(node.ip), + 60000, + { afterId: anchors[env.clients.indexOf(peer)] }, + ).then(() => i))).catch(() => null); + expect(heard, 'no node was told the app had gone').to.not.equal(null); + }); + + it('had the write of the rebuilt app\'s row refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database refused none of the node\'s row writes, so nothing above was tested').to.equal(1); + }); + + it('rebuilds an app whose upgrade completes at its new specification', async function () { + this.timeout(2400000); + const node = env.clients[UPGRADES]; + const atNewSpec = async () => (await localSpec(node, upgrading))?.hash === upgraded.hash; + await driveUntil(node, atNewSpec, { blocks: PASS_BLOCKS, label: 'the node rebuilds the app at its new specification' }); + const installed = await localSpec(node, upgrading); + expect(installed.version, 'the app is not at its new specification version').to.equal(upgraded.spec.version); + await waitFor(async () => Boolean(await getAppContainerId(node.container, upgrading, upgrading)), { + timeout: 120000, interval: 2000, label: 'the rebuilt app runs as a component', + }); + }); +}); From 27c4d0268eb50ce6930eb80f82ca0dda0a12e371 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 12:35:10 +0100 Subject: [PATCH 118/369] test(harness): a seeded version 3 app carries tiered, as every one on the chain does Without it the app's removal refuses its specification as invalid. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/seed-helper.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-infra/runner/framework/seed-helper.js b/test-infra/runner/framework/seed-helper.js index e487705cd4..3666bfc77a 100644 --- a/test-infra/runner/framework/seed-helper.js +++ b/test-infra/runner/framework/seed-helper.js @@ -285,6 +285,8 @@ export async function buildSeedableLegacyApp({ cpu: 0.1, ram: 100, hdd: 1, + // every version 3 specification on the chain carries it, as false + tiered: false, instances, expire, }; From 95d025793f30855b4b9bf10d1ea12663995e8918 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 12:53:42 +0100 Subject: [PATCH 119/369] test(harness): a seeded version 3 app asks for 3 instances, the least its version allows Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/seed-helper.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/framework/seed-helper.js b/test-infra/runner/framework/seed-helper.js index 3666bfc77a..9a8caff70b 100644 --- a/test-infra/runner/framework/seed-helper.js +++ b/test-infra/runner/framework/seed-helper.js @@ -259,7 +259,8 @@ export async function buildSeedableLegacyApp({ // whose chain starts later, and assertAliveOnThisChain refuses it. env = null, height = (env?.initialHeight ?? DEFAULT_INITIAL_HEIGHT) + 10, - instances = 1, + // below version 8 a specification asks for at least 3, as every one on the chain does + instances = 3, owner = null, expire = 22000, allowExternalRepotag = false, From 20402fc54365b59362a52b185144277e42a1de11 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 13:35:26 +0100 Subject: [PATCH 120/369] fix(apps): the test event stream carries each rebuild's outcome and the end of each reinstall pass app:rebuilt names the app and how its rebuild ended; reinstall:passEnded is published once a pass that ran is over. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/advancedWorkflows.js | 1 + ZelBack/src/services/appLifecycle/appRebuild.js | 3 +++ tests/unit/advancedWorkflows.test.js | 10 ++++++++++ tests/unit/appRebuild.test.js | 14 ++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 581b5ed6be..6e4825dc15 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -4630,6 +4630,7 @@ async function reinstallOldApplications() { // loop, on a path that can return or throw from several places, and a leaked // true would make every neighbour stand aside indefinitely. globalState.reinstallationOfOldAppsInProgress = false; + fluxEventBus.publish('reinstall:passEnded', {}); // ASKED ONCE THIS PASS IS OVER, because both of them uninstall and what holds them // apart is the node's install and removal flags: a removal the sweep attempts while // this pass holds those is refused, and goes onto the sweep's backoff rather than diff --git a/ZelBack/src/services/appLifecycle/appRebuild.js b/ZelBack/src/services/appLifecycle/appRebuild.js index 7bc4ff5947..3b103b2518 100644 --- a/ZelBack/src/services/appLifecycle/appRebuild.js +++ b/ZelBack/src/services/appLifecycle/appRebuild.js @@ -4,6 +4,7 @@ * nothing; the callers decide when, and hold the node while it runs. */ const log = require('../../lib/log'); +const fluxEventBus = require('../utils/fluxEventBus'); const appUninstaller = require('./appUninstaller'); const appInstaller = require('./appInstaller'); const { RemovalOutcome } = require('../utils/removalOutcome'); @@ -23,6 +24,7 @@ async function rebuildApp(appSpecs, res) { const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); if (removal !== RemovalOutcome.REMOVED && removal !== RemovalOutcome.NOT_INSTALLED) { log.warn(`appRebuild - ${appSpecs.name} was not removed (${removal}), so it is not reinstalled`); + fluxEventBus.publish('app:rebuilt', { name: appSpecs.name, outcome: InstallOutcome.FAILED }); return InstallOutcome.FAILED; } @@ -41,6 +43,7 @@ async function rebuildApp(appSpecs, res) { await appUninstaller.announceRemoval(appSpecs.name) .catch((error) => log.error(`appRebuild - the removal of ${appSpecs.name} was not announced: ${error.message}`)); } + fluxEventBus.publish('app:rebuilt', { name: appSpecs.name, outcome }); return outcome; } diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 08f827bb1a..4cd28d2c2c 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -7419,6 +7419,16 @@ describe('advancedWorkflows tests', () => { expect(rebuildApp.firstCall.args[0]).to.include({ name: 'myapp', hash: 'newhash', version: 4 }); }); + it('says when the pass has ended', async () => { + sinon.stub(appRebuild, 'rebuildApp').resolves('installed'); + const publish = sinon.stub(fluxEventBus, 'publish'); + dbHelper.findInDatabase.resolves([oldFormat]); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledWith(publish, 'reinstall:passEnded', {}); + }); + it('lets the pass go on to the next app when its rebuild does not complete', async () => { const rebuildApp = sinon.stub(appRebuild, 'rebuildApp').resolves('failed'); const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); diff --git a/tests/unit/appRebuild.test.js b/tests/unit/appRebuild.test.js index 141467fdc8..4bcaa62d9f 100644 --- a/tests/unit/appRebuild.test.js +++ b/tests/unit/appRebuild.test.js @@ -8,6 +8,7 @@ describe('appRebuild tests', () => { let uninstaller; let installer; let appRebuild; + let bus; const spec = { name: 'myapp', version: 8 }; beforeEach(() => { @@ -19,7 +20,9 @@ describe('appRebuild tests', () => { checkAppRequirements: sinon.stub().resolves(true), registerAppLocally: sinon.stub().resolves(InstallOutcome.INSTALLED), }; + bus = { publish: sinon.stub() }; appRebuild = proxyquire('../../ZelBack/src/services/appLifecycle/appRebuild', { + '../utils/fluxEventBus': bus, './appUninstaller': uninstaller, './appInstaller': installer, '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, @@ -79,6 +82,17 @@ describe('appRebuild tests', () => { sinon.assert.notCalled(uninstaller.announceRemoval); }); + it('publishes how each rebuild ended', async () => { + await appRebuild.rebuildApp(spec); + uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); + await appRebuild.rebuildApp(spec); + + expect(bus.publish.args).to.deep.equal([ + ['app:rebuilt', { name: 'myapp', outcome: InstallOutcome.INSTALLED }], + ['app:rebuilt', { name: 'myapp', outcome: InstallOutcome.FAILED }], + ]); + }); + it('settles a removal it could not announce, and still answers', async () => { installer.registerAppLocally.resolves(InstallOutcome.FAILED); uninstaller.announceRemoval.rejects(new Error('no peers')); From b91e49936fb3bc4eb90d848714f1d54276db99e4 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 13:35:26 +0100 Subject: [PATCH 121/369] test(harness): 3307 asks whether the same pass goes on, and whether the network was told The next app's redeploy falls between the failed rebuild and the end of that pass. The node's own spawner may place the app it gave up again, so only the removal heard by another node is asked about. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/node-client.js | 2 ++ ...p-upgraded-from-the-old-format-is-rebuilt.js | 17 ++++++++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index c0e133aa61..11df4b747d 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -217,6 +217,8 @@ export function nodeClient(nodeNum) { // wording of the response stream. 'app:installOutcome', 'app:removed', + 'app:rebuilt', + 'reinstall:passEnded', 'app:componentRedeployed', 'app:specStored', 'app:running', diff --git a/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js b/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js index 4fa16e6278..dec8d44e94 100644 --- a/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js +++ b/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js @@ -121,18 +121,25 @@ describe('an app upgraded from the old format is rebuilt', function () { await env?.teardown(); }); - it('goes on to the next app after one it could not rebuild', async function () { + it('goes on to the next app in the same pass after one it could not rebuild', async function () { this.timeout(2400000); const node = env.clients[FAILS]; - const redeployed = () => waitForComponentRedeployed(node, other, other, false, 0, { afterId: anchors[FAILS] }) - .then(() => true, () => false); - await driveUntil(node, redeployed, { blocks: PASS_BLOCKS, label: 'the pass redeploys the next app' }); + const since = () => node.getEventBuffer().filter((e) => e.id > anchors[FAILS]); + const failedRebuild = () => since().find((e) => e.event === 'app:rebuilt' && e.data.name === failing && e.data.outcome !== 'installed'); + const passEndAfter = (event) => since().find((e) => e.event === 'reinstall:passEnded' && e.id > event.id); + await driveUntil(node, async () => Boolean(failedRebuild() && passEndAfter(failedRebuild())), { + blocks: PASS_BLOCKS, label: 'a reinstall pass rebuilds the app, fails, and ends', + }); + const rebuilt = failedRebuild(); + const ended = passEndAfter(rebuilt); + const redeployed = since().find((e) => e.event === 'app:componentRedeployed' && e.data.name === other + && e.id > rebuilt.id && e.id < ended.id); + expect(redeployed, 'the pass stopped at the app it could not rebuild').to.not.equal(undefined); }); it('tells the network it no longer holds the app it could not rebuild', async function () { this.timeout(120000); const node = env.clients[FAILS]; - expect(await localSpec(node, failing), 'the node still holds the app').to.equal(null); const peers = env.clients.filter((client, i) => client && i !== FAILS); const heard = await Promise.any(peers.map((peer, i) => peer.waitForEvent( 'network:appremoved', From 52c75789f765884c930d45537d475ff9dd9f6c40 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:08:47 +0100 Subject: [PATCH 122/369] refactor(boot): untilSucceeds is a shared utility Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/serviceManager.js | 24 +------------------- ZelBack/src/services/utils/untilSucceeds.js | 25 +++++++++++++++++++++ tests/unit/serviceManager.test.js | 7 +++--- 3 files changed, 29 insertions(+), 27 deletions(-) create mode 100644 ZelBack/src/services/utils/untilSucceeds.js diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 88c3002ee5..f25a6cb35c 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -6,7 +6,7 @@ const https = require('https'); const cacheManager = require('./utils/cacheManager').default; const log = require('../lib/log'); const dbHelper = require('./dbHelper'); -const serviceHelper = require('./serviceHelper'); +const { untilSucceeds } = require('./utils/untilSucceeds'); const explorerService = require('./explorerService'); const fluxCommunication = require('./fluxCommunication'); const networkStateService = require('./networkStateService'); @@ -105,27 +105,6 @@ const dosState = { const portsNotWorking = new Set(); const appsStorageViolations = []; -// How long boot waits before it runs a database step that failed again. -const BOOT_STEP_RETRY_MS = 15 * 1000; - -/** - * Runs `step` until it succeeds, waiting BOOT_STEP_RETRY_MS after each - * failure. Boot holds at a database step that fails rather than starting again - * from the top, so nothing boot already started runs twice. - * @param {string} what Names the step in the log. - * @param {function(): Promise<*>} step - * @returns {Promise<*>} What the step resolved. - */ -async function untilSucceeds(what, step) { - try { - return await step(); - } catch (error) { - log.error(`${what} failed, running it again in ${BOOT_STEP_RETRY_MS / 1000}s: ${error.message}`); - await serviceHelper.delay(BOOT_STEP_RETRY_MS); - return untilSucceeds(what, step); - } -} - /** * Brings every collection boot depends on to its schema: indexes, the merges * their unique indexes need, and the one-off repairs. Every step is safe to @@ -842,7 +821,6 @@ async function startFluxFunctions() { module.exports = { startFluxFunctions, - untilSucceeds, prepareDatabaseSchema, // testing exports dosState, diff --git a/ZelBack/src/services/utils/untilSucceeds.js b/ZelBack/src/services/utils/untilSucceeds.js new file mode 100644 index 0000000000..5dff67b39c --- /dev/null +++ b/ZelBack/src/services/utils/untilSucceeds.js @@ -0,0 +1,25 @@ +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); + +// How long a boot step waits before it runs again after a failure. +const BOOT_STEP_RETRY_MS = 15 * 1000; + +/** + * Runs `step` until it succeeds, waiting BOOT_STEP_RETRY_MS after each + * failure. Boot holds at a database step that fails rather than starting again + * from the top, so nothing boot already started runs twice. + * @param {string} what Names the step in the log. + * @param {function(): Promise<*>} step + * @returns {Promise<*>} What the step resolved. + */ +async function untilSucceeds(what, step) { + try { + return await step(); + } catch (error) { + log.error(`${what} failed, running it again in ${BOOT_STEP_RETRY_MS / 1000}s: ${error.message}`); + await serviceHelper.delay(BOOT_STEP_RETRY_MS); + return untilSucceeds(what, step); + } +} + +module.exports = { untilSucceeds, BOOT_STEP_RETRY_MS }; diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index af7d7ba31b..5ba74fcc61 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -5,14 +5,13 @@ const sinon = require('sinon'); chai.use(chaiAsPromised); const { expect } = chai; -const { - untilSucceeds, prepareDatabaseSchema, dosState, -} = require('../../ZelBack/src/services/serviceManager'); +const { prepareDatabaseSchema, dosState } = require('../../ZelBack/src/services/serviceManager'); +const { untilSucceeds } = require('../../ZelBack/src/services/utils/untilSucceeds'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); -describe('serviceManager untilSucceeds', () => { +describe('untilSucceeds', () => { afterEach(() => sinon.restore()); it('runs a failing step again after a wait, until it succeeds, and answers what it resolved', async () => { From 8fac3a7821154f07ff9f9accfa8d9bc7c0674a58 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:08:47 +0100 Subject: [PATCH 123/369] fix(apps): boot never starts an app whose location it cannot confirm An app whose location record has expired, or cannot be read, may have been placed elsewhere while the node was down: boot removes it, and tells the network when the record could not be read. A removal that does not finish holds the app as departing, so it is neither started nor announced as running, and is run again until it does. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 56 +++++++++++++------ tests/unit/appStartupManager.test.js | 47 +++++++++++----- 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 1ae777d88f..2f70f99e5c 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -22,6 +22,8 @@ const fluxEventBus = require('../utils/fluxEventBus'); const nodeConfirmationService = require('../nodeConfirmationService'); const { localAppsInformation, SIGTERM_EXPIRY_MS, RUNNING_EXPIRY_MS } = require('../utils/appConstants'); const { getNonGComponentIdentifiers, parseContainerName, appHasValidLocationOnNode } = require('../utils/appUtilities'); +const { RemovalOutcome } = require('../utils/removalOutcome'); +const { untilSucceeds } = require('../utils/untilSucceeds'); const SYNC_TIMEOUT_MS = config.system.bootSyncTimeoutMs ?? 300000; @@ -113,6 +115,32 @@ async function openInstalledAppPorts() { } } + +/** + * Removes an app boot may not start. A removal that does not finish holds the app as + * departing - never started by the reconciler, never announced as running - and is + * run again until it does. + * @param {string} appName + * @param {boolean} announce Whether the removal is told to the network. + * @returns {Promise} Whether the app is gone now. + */ +async function removeAppBootMayNotStart(appName, announce) { + const remove = async () => { + const outcome = await appUninstaller.removeAppLocally(appName, null, true, true, announce); + if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + throw new Error(`${appName} was not removed (${outcome})`); + } + }; + try { + await remove(); + return true; + } catch (error) { + log.warn(`appStartupManager - ${error.message}; held stopped and unannounced until it is removed`); + globalState.departingApps.enter(appName); + untilSucceeds(`Removal of ${appName}`, remove).then(() => globalState.departingApps.leave(appName)); + return false; + } +} /** * Reconcile local app state against the network on boot. For each stopped * container, checks whether this node still has a valid location record. @@ -222,24 +250,16 @@ async function reconcileAppsOnBoot() { if (localSocketAddr) { // eslint-disable-next-line no-await-in-loop const hasValidLocation = await appHasValidLocationOnNode(appName, localSocketAddr); - if (hasValidLocation === null) { - // Neither started nor removed on an unread location: the reconciler decides later. - log.warn(`appStartupManager - App ${appName}: its location record could not be read, leaving it as it is`); - results.appsFailed.push({ app: appName, error: 'location record unreadable' }); - // eslint-disable-next-line no-continue - continue; - } - if (!hasValidLocation) { - log.warn(`appStartupManager - App ${appName} no longer has a valid location record for this node (${localSocketAddr}), removing locally`); - try { - // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appName, null, true, true, false); - results.appsRemoved.push(appName); - log.info(`appStartupManager - App ${appName} removed locally (was reassigned to another node)`); - } catch (removeError) { - log.error(`appStartupManager - Failed to remove app ${appName}: ${removeError.message}`); - results.appsFailed.push({ app: appName, error: removeError.message }); - } + if (hasValidLocation !== true) { + // An app whose location this node holds no longer, or cannot read, may have been + // placed elsewhere while this node was down: it is removed, never started. One it + // cannot confirm is announced as removed; an expired one the network has dropped. + const unconfirmed = hasValidLocation === null; + log.warn(`appStartupManager - App ${appName}: ${unconfirmed ? 'its location record could not be read' : `no valid location record for this node (${localSocketAddr})`}, removing locally`); + // eslint-disable-next-line no-await-in-loop + const removed = await removeAppBootMayNotStart(appName, unconfirmed); + if (removed) results.appsRemoved.push(appName); + else results.appsFailed.push({ app: appName, error: 'not removed; held stopped until it is' }); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(2000); // eslint-disable-next-line no-continue diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 3c2afb9c75..f75abbc89b 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -59,7 +59,7 @@ describe('appStartupManager tests', () => { }; appUninstallerStub = { - removeAppLocally: sinon.stub().resolves(), + removeAppLocally: sinon.stub().resolves('removed'), }; const mockDb = { db: sinon.stub().returns('mockDatabase') }; @@ -97,6 +97,18 @@ describe('appStartupManager tests', () => { '../appMonitoring/appReconciler': appReconcilerStub, './appUninstaller': appUninstallerStub, '../utils/globalState': globalStateStub, + // runs the step again at once until it succeeds, as the real one does after a wait + '../utils/untilSucceeds': { + untilSucceeds: async (what, step) => { + for (;;) { + try { + return await step(); // eslint-disable-line no-await-in-loop + } catch { + // run it again + } + } + }, + }, '../appQuery/appQueryService': appQueryServiceStub, '../utils/appConstants': { localAppsInformation: 'localAppsInformation', SIGTERM_EXPIRY_MS: 420000, RUNNING_EXPIRY_MS: 7500000 }, '../utils/appUtilities': appUtilities, @@ -313,28 +325,33 @@ describe('appStartupManager tests', () => { expect(results.appsRemoved).to.deep.equal(['AppB']); }); - it('should record failure when removeAppLocally throws', async () => { + it('holds an app it could not remove, never starting it, and removes it once it can', async () => { dockerServiceStub.dockerListContainers.resolves([ { Names: ['/fluxAppA'], State: 'exited' }, ]); dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); - // Expired location dbHelperStub.findInDatabase.onSecondCall().resolves([]); - - appUninstallerStub.removeAppLocally.rejects(new Error('Remove failed')); + let heldWhileRetrying; + appUninstallerStub.removeAppLocally.onFirstCall().resolves('failed'); + appUninstallerStub.removeAppLocally.onSecondCall().callsFake(async () => { + heldWhileRetrying = globalStateStub.departingApps.has('AppA'); + return 'removed'; + }); const results = await appStartupManager.reconcileAppsOnBoot(); + await new Promise((resolve) => { setImmediate(resolve); }); - expect(results.appsRemoved).to.deep.equal([]); - expect(results.appsFailed).to.have.lengthOf(1); - expect(results.appsFailed[0].app).to.equal('AppA'); - expect(results.appsFailed[0].error).to.equal('Remove failed'); + expect(results.appsFailed.map((failure) => failure.app)).to.deep.equal(['AppA']); + expect(appReconcilerStub.enqueue.calledWith('AppA'), 'an app boot could not remove was started').to.equal(false); + expect(heldWhileRetrying, 'the app was not held while its removal was retried').to.equal(true); + sinon.assert.calledTwice(appUninstallerStub.removeAppLocally); + expect(globalStateStub.departingApps.has('AppA'), 'the hold outlived the removal').to.equal(false); }); - it('neither starts nor removes an app whose location record cannot be read', async () => { - // Started, an app whose location expired runs beside the instance placed in - // its stead; removed, a valid one is lost. The reconciler decides it later. + it('removes an app whose location record cannot be read, and tells the network', async () => { + // Its location may have been given to another node while this one was down, so + // it is never started unconfirmed. dockerServiceStub.dockerListContainers.resolves([ { Names: ['/fluxAppA'], State: 'exited' }, ]); @@ -343,9 +360,9 @@ describe('appStartupManager tests', () => { const results = await appStartupManager.reconcileAppsOnBoot(); - expect(results.appsStarted).to.deep.equal([]); - expect(results.appsRemoved).to.deep.equal([]); - expect(results.appsFailed.map((failure) => failure.app)).to.deep.equal(['AppA']); + sinon.assert.calledOnceWithExactly(appUninstallerStub.removeAppLocally, 'AppA', null, true, true, true); + expect(results.appsRemoved).to.deep.equal(['AppA']); + expect(appReconcilerStub.enqueue.calledWith('AppA')).to.equal(false); }); it('should check location after g: syncthing check', async () => { From 80513af416a6186e638333a1455021455aa0ae0d Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:10:03 +0100 Subject: [PATCH 124/369] test(harness): 3308, boot removes an app whose location it cannot confirm mongo-control.failCommands fails any commands on one collection of one node, a set number of times or until released; failWrites is its writes. 3308 refuses a node's location reads across a FluxOS restart: boot removes the app it holds and the network is told. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/mongo-control.js | 19 ++-- ...8-boot-removes-an-app-it-cannot-confirm.js | 106 ++++++++++++++++++ 2 files changed, 118 insertions(+), 7 deletions(-) create mode 100644 test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js index ad7075baed..abe213131c 100644 --- a/test-infra/runner/framework/mongo-control.js +++ b/test-infra/runner/framework/mongo-control.js @@ -55,16 +55,17 @@ export async function releaseCommands() { }); } -// The next `times` writes to one collection of one node fail, as a write the -// database could not complete does, and every other node's go through: each node -// keeps its data in databases of its own (dbClient's prefix). Resolves to a -// release that turns the failure off and resolves to how many writes it failed. -export async function failWrites(nodeNum, database, collection, { times = 1 } = {}) { +// The given commands on one collection of one node fail, as a call the database could +// not complete does, and every other node's go through: each node keeps its data in +// databases of its own (dbClient's prefix). `times` fails that many and then stops; left +// out, every one fails until released. Resolves to a release that turns the failure off +// and resolves to how many commands it failed. +export async function failCommands(nodeNum, database, collection, { commands, times } = {}) { const namespace = `node${String(nodeNum).padStart(2, '0')}_${database}.${collection}`; await withAdmin((admin) => admin.command({ configureFailPoint: 'failCommand', - mode: { times }, - data: { failCommands: ['insert', 'update'], namespace, errorCode: 91 }, + mode: times === undefined ? 'alwaysOn' : { times }, + data: { failCommands: commands, namespace, errorCode: 91 }, })); return async function release() { // turning a failpoint off answers how many times it fired @@ -73,3 +74,7 @@ export async function failWrites(nodeNum, database, collection, { times = 1 } = }; } +// The next `times` writes to one collection of one node fail (failCommands). +export async function failWrites(nodeNum, database, collection, { times = 1 } = {}) { + return failCommands(nodeNum, database, collection, { commands: ['insert', 'update'], times }); +} diff --git a/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js new file mode 100644 index 0000000000..74836995d4 --- /dev/null +++ b/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js @@ -0,0 +1,106 @@ +/* + * At boot a node reads the network's location record of each app it holds: one whose + * record has expired was placed elsewhere while the node was down. When the record + * cannot be read, the node cannot tell, so it never starts the app: it removes it and + * tells the network. + * + * Every FluxOS start runs that check, so a restart is a boot here. The node's reads of + * the location collection are refused for the whole of it. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { getAppContainerId, restartFluxos } from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const HOLDER = 0; + +function step(what) { + console.log(`# 3308 ${new Date().toISOString()} ${what}`); +} + +describe('boot removes an app whose location it cannot confirm', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2ebootunread${Date.now()}`; + let holder; + let release; + let anchors; + let failedReads; + let heldAfterBoot; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 5, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ + env, + name: appName, + compose: [{ + name: appName, + description: 'boot location test container', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + await installOnNodes(env, app, [HOLDER]); + await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)), { + timeout: 120000, interval: 2000, label: 'the app runs before the restart', + }); + + anchors = env.clients.map((client) => client?.getLastEventId()); + release = await failCommands(HOLDER + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); + step('restarting FluxOS with its location reads refused'); + await restartFluxos(holder.container, { readyTimeoutMs: 180000 }); + await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + // read at boot's end: the node's own spawner may place the app again later + heldAfterBoot = (await holder.getInstalledApps()).data?.some((a) => a.name === appName) ?? null; + failedReads = await release(); + release = null; + step('booted'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('no longer holds the app once boot is over', function () { + expect(heldAfterBoot, 'boot left an app it could not confirm installed').to.equal(false); + }); + + it('tells the network it no longer holds the app', async function () { + this.timeout(90000); + const peers = env.clients.filter((client, i) => client && i !== HOLDER); + const heard = await Promise.any(peers.map((peer) => peer.waitForEvent( + 'network:appremoved', + (d) => d.name === appName && d.ip.startsWith(holder.ip), + 60000, + { afterId: anchors[env.clients.indexOf(peer)] }, + ))).catch(() => null); + expect(heard, 'no node was told the app had gone').to.not.equal(null); + }); +}); From f07bf250b4ea5a8468082663acd5e923a16a28bd Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:22:09 +0100 Subject: [PATCH 125/369] test(harness): 3308 boots the node with the app's container stopped Boot checks the location of apps whose containers are stopped, as a machine reboot leaves them. The node's FluxOS is killed and held down, the app's container is stopped, and the location reads are refused before FluxOS comes back. A canary checks the container was stopped at boot. Co-Authored-By: Claude Opus 5.5 --- ...8-boot-removes-an-app-it-cannot-confirm.js | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js index 74836995d4..f92b6732f9 100644 --- a/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js @@ -4,8 +4,9 @@ * cannot be read, the node cannot tell, so it never starts the app: it removes it and * tells the network. * - * Every FluxOS start runs that check, so a restart is a boot here. The node's reads of - * the location collection are refused for the whole of it. + * The check runs for apps whose containers are stopped, which is what a machine reboot + * leaves. Here FluxOS is killed and held down, the app's container is stopped, and the + * node's reads of the location collection are refused before FluxOS comes back. */ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; @@ -14,7 +15,9 @@ import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; -import { getAppContainerId, restartFluxos } from '../framework/container.js'; +import { + crashFluxos, getAppContainerId, isAppContainerRunning, releaseFluxos, stopAppContainer, +} from '../framework/container.js'; import { waitFor } from '../framework/wait.js'; import { failCommands } from '../framework/mongo-control.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -33,6 +36,7 @@ describe('boot removes an app whose location it cannot confirm', function () { let release; let anchors; let failedReads; + let stoppedAtBoot; let heldAfterBoot; before(async function () { @@ -66,10 +70,14 @@ describe('boot removes an app whose location it cannot confirm', function () { timeout: 120000, interval: 2000, label: 'the app runs before the restart', }); + step('taking the node down, its app container stopped and its location reads refused'); + await crashFluxos(holder.container, { hold: true }); + const stopped = await stopAppContainer(holder.container, appName, appName); + if (stopped.exitCode !== 0) throw new Error(`could not stop the app container: ${stopped.output}`); + stoppedAtBoot = !(await isAppContainerRunning(holder.container, appName)); anchors = env.clients.map((client) => client?.getLastEventId()); release = await failCommands(HOLDER + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); - step('restarting FluxOS with its location reads refused'); - await restartFluxos(holder.container, { readyTimeoutMs: 180000 }); + await releaseFluxos(holder.container, { readyTimeoutMs: 180000 }); await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); // read at boot's end: the node's own spawner may place the app again later heldAfterBoot = (await holder.getInstalledApps()).data?.some((a) => a.name === appName) ?? null; @@ -84,6 +92,10 @@ describe('boot removes an app whose location it cannot confirm', function () { await env?.teardown(); }); + it('boots with the app\'s container stopped', function () { + expect(stoppedAtBoot, 'the app\'s container was running when FluxOS came back, so boot had nothing to check').to.equal(true); + }); + it('had its reads of the location records refused by the database', function () { expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); }); From 8b3e82d16edec10b22dc8869a68ee45ca5d285ee Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:31:46 +0100 Subject: [PATCH 126/369] test(harness): 3309, a removal that cannot delete the app's row keeps the app The node's delete of the row is refused once during a hard redeploy's removal. The removal fails, the reconciler rebuilds the app from the row, and the node still holds it after any later delete could have run. Co-Authored-By: Claude Opus 5.5 --- ...hat-cannot-delete-the-row-keeps-the-app.js | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js diff --git a/test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js b/test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js new file mode 100644 index 0000000000..49169d1af5 --- /dev/null +++ b/test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js @@ -0,0 +1,110 @@ +/* + * A hard redeploy whose removal cannot delete the app's row fails, and the row stays, + * still describing the app. Nothing deletes it later: the reconciler rebuilds the app + * from it, and the node holds the app from then on. + * + * The node's first delete of a row is refused once, during the redeploy's removal. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { getAppContainerId, isAppContainerRunning } from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const HOLDER = 0; +// longer than any delay a background delete of the row could be scheduled at +const SETTLE_MS = 90000; + +function step(what) { + console.log(`# 3309 ${new Date().toISOString()} ${what}`); +} + +describe('a removal that cannot delete the app\'s row keeps the app', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2erowkept${Date.now()}`; + let holder; + let release; + let containerBefore; + let redeployedAt; + let failedDeletes; + + const holds = async () => (await holder.getInstalledApps()).data?.some((a) => a.name === appName) ?? null; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 3, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ + env, + name: appName, + compose: [{ + name: appName, + description: 'row kept test container', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + await installOnNodes(env, app, [HOLDER]); + await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)), { + timeout: 120000, interval: 2000, label: 'the app runs before the redeploy', + }); + containerBefore = await getAppContainerId(holder.container, appName, appName); + + const auth = await authenticate(holder.url, appOwnerKey()); + release = await failCommands(HOLDER + 1, 'localzelapps', 'zelappsinformation', { commands: ['findAndModify'], times: 1 }); + step('hard redeploy with the row delete refused once'); + const body = await holder.redeployApp(appName, auth.zelidauth, { force: true }); + redeployedAt = Date.now(); + if (/Unauthorized/i.test(body)) throw new Error(`hard redeploy refused: ${body.slice(0, 400)}`); + failedDeletes = await release(); + release = null; + step('redeploy returned'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('had its delete of the app\'s row refused by the database', function () { + expect(failedDeletes, 'the database refused none of the node\'s row deletes, so nothing below was tested').to.equal(1); + }); + + it('rebuilds the app from the row it kept', async function () { + this.timeout(180000); + await waitFor(async () => { + const id = await getAppContainerId(holder.container, appName, appName); + return Boolean(id) && id !== containerBefore && isAppContainerRunning(holder.container, appName); + }, { timeout: 150000, interval: 2000, label: 'the app runs again in a new container' }); + }); + + it('still holds the app once the removal is long over', async function () { + this.timeout(SETTLE_MS + 60000); + const wait = SETTLE_MS - (Date.now() - redeployedAt); + if (wait > 0) await new Promise((resolve) => { setTimeout(resolve, wait); }); + expect(await holds(), 'the app\'s row was deleted after its removal had failed').to.equal(true); + expect(await isAppContainerRunning(holder.container, appName), 'the app no longer runs').to.equal(true); + }); +}); From 4bae3fc1b608df9ced276a9268adb0fc69af7d7a Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:53:49 +0100 Subject: [PATCH 127/369] fix(apps): a removal that stops part way is held as departing and run again until it finishes Once a whole app's teardown has begun, a failure leaves the app part removed: its row can outlive its containers and volume. appUninstaller.finishRemoval holds it as departing, so the reconciler never starts it and announcements leave it out, and runs the removal again until it finishes, telling the network then unless it was already told. Boot hands an app it could not remove to the same routine. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 20 +---- .../services/appLifecycle/appUninstaller.js | 54 +++++++++++-- tests/unit/appStartupManager.test.js | 15 +--- tests/unit/appUninstaller.test.js | 80 ++++++++++++++----- 4 files changed, 117 insertions(+), 52 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 2f70f99e5c..17b27de5b7 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -23,7 +23,6 @@ const nodeConfirmationService = require('../nodeConfirmationService'); const { localAppsInformation, SIGTERM_EXPIRY_MS, RUNNING_EXPIRY_MS } = require('../utils/appConstants'); const { getNonGComponentIdentifiers, parseContainerName, appHasValidLocationOnNode } = require('../utils/appUtilities'); const { RemovalOutcome } = require('../utils/removalOutcome'); -const { untilSucceeds } = require('../utils/untilSucceeds'); const SYNC_TIMEOUT_MS = config.system.bootSyncTimeoutMs ?? 300000; @@ -125,21 +124,10 @@ async function openInstalledAppPorts() { * @returns {Promise} Whether the app is gone now. */ async function removeAppBootMayNotStart(appName, announce) { - const remove = async () => { - const outcome = await appUninstaller.removeAppLocally(appName, null, true, true, announce); - if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { - throw new Error(`${appName} was not removed (${outcome})`); - } - }; - try { - await remove(); - return true; - } catch (error) { - log.warn(`appStartupManager - ${error.message}; held stopped and unannounced until it is removed`); - globalState.departingApps.enter(appName); - untilSucceeds(`Removal of ${appName}`, remove).then(() => globalState.departingApps.leave(appName)); - return false; - } + const outcome = await appUninstaller.removeAppLocally(appName, null, true, true, announce); + if (outcome === RemovalOutcome.REMOVED || outcome === RemovalOutcome.NOT_INSTALLED) return true; + appUninstaller.finishRemoval(appName, true, announce); + return false; } /** * Reconcile local app state against the network on boot. For each stopped diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index ba22d3735a..68544c64ab 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -25,6 +25,7 @@ const volumeService = require('../utils/volumeService'); const fluxEventBus = require('../utils/fluxEventBus'); const { Privilege, authOf } = require('../utils/privileges'); const { RemovalOutcome } = require('../utils/removalOutcome'); +const { untilSucceeds, BOOT_STEP_RETRY_MS } = require('../utils/untilSucceeds'); /** * The node does not hold this application. @@ -57,6 +58,36 @@ function setOnComponentRemoved(callback) { onComponentRemoved = callback; } +// Apps whose removal stopped part way, each being run again until it finishes. +const unfinishedRemovals = new Set(); + +/** + * Runs the removal of an app that stopped part way again until it finishes. Until + * then the app is held as departing: the reconciler never starts it and the node's + * announcements leave it out. A call for an app already held does nothing. + * @param {string} appName + * @param {boolean} force Passed to each removal. + * @param {boolean} announce Whether the removal that finishes tells the network. + * @returns {void} + */ +function finishRemoval(appName, force, announce) { + if (unfinishedRemovals.has(appName)) return; + unfinishedRemovals.add(appName); + globalState.departingApps.enter(appName); + log.warn(`${appName} was not fully removed; held stopped and unannounced until it is`); + // first run once the removal that stopped has released the node's removal lock + serviceHelper.delay(BOOT_STEP_RETRY_MS).then(() => untilSucceeds(`Removal of ${appName}`, async () => { + // eslint-disable-next-line no-use-before-define + const outcome = await removeAppLocally(appName, null, force, true, announce); + if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + throw new Error(`${appName} was not removed (${outcome})`); + } + })).then(() => { + unfinishedRemovals.delete(appName); + globalState.departingApps.leave(appName); + }); +} + /** * Tells the network this node no longer holds the app (fluxappremoved), and drops it * from the running apps cache. For an app whose removal from this node is complete. @@ -879,6 +910,10 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // the same finally, and a refused duplicate must not release the lock the removal // actually running is holding. let acquired = false; + // Set once the teardown of a whole app has begun: a failure from there on leaves + // the app part removed. + let tornDownApp = null; + let announced = false; try { // Normalise to the bare identifier this function reasons about: a caller may // pass the flux-prefixed docker name (e.g. the syncthing flow), which would @@ -981,6 +1016,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen appSpecifications = await checkAndDecryptAppSpecs(appSpecifications); appSpecifications = specificationFormatter(appSpecifications); + if (!isComponent) tornDownApp = appName; if (appSpecifications.version >= 4 && !isComponent) { // it is a composed application // eslint-disable-next-line no-restricted-syntax @@ -1017,7 +1053,10 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen fluxEventBus.publish('app:removed', { name: appName }); - if (sendMessage) await announceRemoval(appName); + if (sendMessage) { + await announceRemoval(appName); + announced = true; + } if (!isComponent) { const dockerNetworkStatus = { @@ -1140,12 +1179,12 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen } } // A specification that cannot be found anywhere means the node is not holding - // this app, which answers a caller asking for it to be gone. Every other error - // leaves it possibly here, whole or in part, and only the caller can decide what - // that costs it. - return error instanceof AppNotFoundError - ? RemovalOutcome.NOT_INSTALLED - : RemovalOutcome.FAILED; + // this app, which answers a caller asking for it to be gone. A whole app whose + // teardown had begun is finished in the background, and the network is told + // once it is gone. Any other error leaves the app as it was. + if (error instanceof AppNotFoundError) return RemovalOutcome.NOT_INSTALLED; + if (tornDownApp) finishRemoval(tornDownApp, force, !announced); + return RemovalOutcome.FAILED; } finally { if (acquired) globalState.removalInProgress = false; if (departingName) { @@ -1339,6 +1378,7 @@ async function removeAppLocallyApi(req, res) { module.exports = { announceRemoval, + finishRemoval, hardUninstallComponent, hardUninstallApplication, softUninstallComponent, diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index f75abbc89b..5f74bb0d88 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -60,6 +60,7 @@ describe('appStartupManager tests', () => { appUninstallerStub = { removeAppLocally: sinon.stub().resolves('removed'), + finishRemoval: sinon.stub(), }; const mockDb = { db: sinon.stub().returns('mockDatabase') }; @@ -325,28 +326,20 @@ describe('appStartupManager tests', () => { expect(results.appsRemoved).to.deep.equal(['AppB']); }); - it('holds an app it could not remove, never starting it, and removes it once it can', async () => { + it('hands an app it could not remove to be finished, never starting it', async () => { dockerServiceStub.dockerListContainers.resolves([ { Names: ['/fluxAppA'], State: 'exited' }, ]); dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); // Expired location dbHelperStub.findInDatabase.onSecondCall().resolves([]); - let heldWhileRetrying; - appUninstallerStub.removeAppLocally.onFirstCall().resolves('failed'); - appUninstallerStub.removeAppLocally.onSecondCall().callsFake(async () => { - heldWhileRetrying = globalStateStub.departingApps.has('AppA'); - return 'removed'; - }); + appUninstallerStub.removeAppLocally.resolves('failed'); const results = await appStartupManager.reconcileAppsOnBoot(); - await new Promise((resolve) => { setImmediate(resolve); }); expect(results.appsFailed.map((failure) => failure.app)).to.deep.equal(['AppA']); expect(appReconcilerStub.enqueue.calledWith('AppA'), 'an app boot could not remove was started').to.equal(false); - expect(heldWhileRetrying, 'the app was not held while its removal was retried').to.equal(true); - sinon.assert.calledTwice(appUninstallerStub.removeAppLocally); - expect(globalStateStub.departingApps.has('AppA'), 'the hold outlived the removal').to.equal(false); + sinon.assert.calledOnceWithExactly(appUninstallerStub.finishRemoval, 'AppA', true, false); }); it('removes an app whose location record cannot be read, and tells the network', async () => { diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 59fc9c6c2d..0885e8821c 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -723,34 +723,78 @@ describe('appUninstaller tests', () => { const res = null; // exercised without a response stream describe('a cleanup step that fails after the containers are gone', () => { - // The row stays and still describes the app, so the reconciler rebuilds it: a - // removal is never finished later, against a row a reinstall may have written. - it('fails the removal and keeps the row when the row cannot be deleted', async () => { - const deleteRow = sinon.stub().rejects(new Error('connection timed out')); + // The app is part removed: it is held as departing - never started, never + // announced - and its removal runs again until it finishes, which tells the network. + const settle = async (done) => { + for (let i = 0; i < 200 && !done(); i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 10); }); + } + }; + + it('fails the removal, holds the app, and finishes the removal once the row can be deleted', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + let heldWhileFinishing = null; + deleteRow.onSecondCall().callsFake(async () => { + heldWhileFinishing = globalStateStub.departingApps.has('testapp'); + }); const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); - const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); - try { - const outcome = await uninstaller.removeAppLocally('testapp', res, true); - await clock.tickAsync(60 * 60 * 1000); + const outcome = await uninstaller.removeAppLocally('testapp', res, true); + await settle(() => !globalStateStub.departingApps.has('testapp') && deleteRow.calledTwice); - expect(outcome).to.equal('failed'); - sinon.assert.calledOnce(deleteRow); - } finally { - clock.restore(); - } - expect(globalStateStub.departingApps.has('testapp'), 'left the app held as departing').to.be.false; + expect(outcome).to.equal('failed'); + expect(heldWhileFinishing, 'the part-removed app was not held while its removal was run again').to.be.true; + sinon.assert.calledTwice(deleteRow); + expect(globalStateStub.departingApps.has('testapp'), 'the hold outlived the removal').to.be.false; + sinon.assert.calledOnce(getLocalSocketAddressStub); }); - it('fails the removal, before the row, when the runtime state cannot be removed', async () => { + it('holds an app the network was already told of, and does not tell it twice', async () => { const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + let heldWhileFinishing = null; + deleteRow.onSecondCall().callsFake(async () => { + heldWhileFinishing = globalStateStub.departingApps.has('testapp'); + }); const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); - runtimeStateStub.remove.rejects(new Error('connection timed out')); - const outcome = await uninstaller.removeAppLocally('testapp', res, true); + await uninstaller.removeAppLocally('testapp', res, true, true, true); + await settle(() => !globalStateStub.departingApps.has('testapp') && deleteRow.calledTwice); + + expect(heldWhileFinishing, 'the part-removed app was not held while its removal was run again').to.be.true; + sinon.assert.calledOnce(getLocalSocketAddressStub); + }); + + it('fails the removal, before the row, when the runtime state cannot be removed, and finishes it', async () => { + const deleteRow = sinon.stub().resolves(); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + runtimeStateStub.remove.onFirstCall().rejects(new Error('connection timed out')); + const outcome = await uninstaller.removeAppLocally('testapp', res, true); expect(outcome).to.equal('failed'); - sinon.assert.notCalled(deleteRow); + await settle(() => deleteRow.called); + + sinon.assert.calledOnce(deleteRow); + }); + + it('runs one removal again for an app however often it is handed over', async () => { + let finishRow; + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + deleteRow.onSecondCall().returns(new Promise((resolve) => { finishRow = resolve; })); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + + await uninstaller.removeAppLocally('testapp', res, true); + await settle(() => deleteRow.calledTwice); + uninstaller.finishRemoval('testapp', true, true); + finishRow(); + await settle(() => !globalStateStub.departingApps.has('testapp')); + // time for a second run, had one been started, to reach the row + await new Promise((resolve) => { setTimeout(resolve, 100); }); + + sinon.assert.calledTwice(deleteRow); }); }); From b83a40f4ade9225d6811abef296122a756d03e8a Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 14:55:42 +0100 Subject: [PATCH 128/369] test(harness): 3309 holds a part-removed app unannounced and finishes its removal The node's deletes of the row are refused four times during a hard redeploy's removal and its next runs. The node announces a second app while the first is held but never the held one, then no longer holds it and tells the network. Co-Authored-By: Claude Opus 5.5 --- ...hat-cannot-delete-the-row-keeps-the-app.js | 110 --------------- ...removal-that-stops-part-way-is-finished.js | 130 ++++++++++++++++++ 2 files changed, 130 insertions(+), 110 deletions(-) delete mode 100644 test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js create mode 100644 test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js diff --git a/test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js b/test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js deleted file mode 100644 index 49169d1af5..0000000000 --- a/test-infra/runner/tests/3309-a-removal-that-cannot-delete-the-row-keeps-the-app.js +++ /dev/null @@ -1,110 +0,0 @@ -/* - * A hard redeploy whose removal cannot delete the app's row fails, and the row stays, - * still describing the app. Nothing deletes it later: the reconciler rebuilds the app - * from it, and the node holds the app from then on. - * - * The node's first delete of a row is refused once, during the redeploy's removal. - */ -import { describe, it, before, after } from 'mocha'; -import { expect } from 'chai'; -import { createTestEnv } from '../framework/test-env.js'; -import { pushImage } from '../framework/registry-helper.js'; -import { buildSeedableApp } from '../framework/seed-helper.js'; -import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; -import { getAppContainerId, isAppContainerRunning } from '../framework/container.js'; -import { waitFor } from '../framework/wait.js'; -import { failCommands } from '../framework/mongo-control.js'; -import { authenticate } from '../auth.js'; -import { appOwnerKey } from '../framework/keys.js'; -import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; - -const HOLDER = 0; -// longer than any delay a background delete of the row could be scheduled at -const SETTLE_MS = 90000; - -function step(what) { - console.log(`# 3309 ${new Date().toISOString()} ${what}`); -} - -describe('a removal that cannot delete the app\'s row keeps the app', function () { - let env; - dumpLogsOnFailure(() => env); - const appName = `e2erowkept${Date.now()}`; - let holder; - let release; - let containerBefore; - let redeployedAt; - let failedDeletes; - - const holds = async () => (await holder.getInstalledApps()).data?.some((a) => a.name === appName) ?? null; - - before(async function () { - this.timeout(600000); - env = await createTestEnv({ hookCtx: this, nodes: 3, tickerAutostart: false }); - await bootAndPeer(env); - holder = env.clients[HOLDER]; - - await pushImage(appName, 'v1'); - const app = await buildSeedableApp({ - env, - name: appName, - compose: [{ - name: appName, - description: 'row kept test container', - repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, - ports: [], - domains: [''], - environmentParameters: [], - commands: [], - containerPorts: [80], - containerData: '/tmp', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', - }], - }); - await installOnNodes(env, app, [HOLDER]); - await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)), { - timeout: 120000, interval: 2000, label: 'the app runs before the redeploy', - }); - containerBefore = await getAppContainerId(holder.container, appName, appName); - - const auth = await authenticate(holder.url, appOwnerKey()); - release = await failCommands(HOLDER + 1, 'localzelapps', 'zelappsinformation', { commands: ['findAndModify'], times: 1 }); - step('hard redeploy with the row delete refused once'); - const body = await holder.redeployApp(appName, auth.zelidauth, { force: true }); - redeployedAt = Date.now(); - if (/Unauthorized/i.test(body)) throw new Error(`hard redeploy refused: ${body.slice(0, 400)}`); - failedDeletes = await release(); - release = null; - step('redeploy returned'); - }); - - after(async function () { - this.timeout(60000); - await release?.().catch(() => {}); - await env?.teardown(); - }); - - it('had its delete of the app\'s row refused by the database', function () { - expect(failedDeletes, 'the database refused none of the node\'s row deletes, so nothing below was tested').to.equal(1); - }); - - it('rebuilds the app from the row it kept', async function () { - this.timeout(180000); - await waitFor(async () => { - const id = await getAppContainerId(holder.container, appName, appName); - return Boolean(id) && id !== containerBefore && isAppContainerRunning(holder.container, appName); - }, { timeout: 150000, interval: 2000, label: 'the app runs again in a new container' }); - }); - - it('still holds the app once the removal is long over', async function () { - this.timeout(SETTLE_MS + 60000); - const wait = SETTLE_MS - (Date.now() - redeployedAt); - if (wait > 0) await new Promise((resolve) => { setTimeout(resolve, wait); }); - expect(await holds(), 'the app\'s row was deleted after its removal had failed').to.equal(true); - expect(await isAppContainerRunning(holder.container, appName), 'the app no longer runs').to.equal(true); - }); -}); diff --git a/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js b/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js new file mode 100644 index 0000000000..a879181a9f --- /dev/null +++ b/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js @@ -0,0 +1,130 @@ +/* + * A hard redeploy whose removal cannot delete the app's row stops with the app part + * removed: its containers and volume are gone, its row is not. The node holds it as + * departing - never announced - and runs the removal again until it finishes, then + * tells the network it no longer holds the app. + * + * The node's deletes of the row are refused REFUSED times: the redeploy's removal and + * the next runs of it, so the app is held across at least one announcement. A second + * app on the node is in every announcement, which shows announcements went out. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { getAppContainerId } from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const HOLDER = 0; +// each run of the removal waits 15s after the last, and the node announces every ~30s +const REFUSED = 4; + +function step(what) { + console.log(`# 3309 ${new Date().toISOString()} ${what}`); +} + +const component = (name) => ({ + name, + description: 'part removal test container', + repotag: `${REGISTRY_REPO_HOST}/${name}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', +}); + +describe('a removal that stops part way is finished', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + const appName = `e2epartremoved${stamp}`; + const otherName = `e2epartother${stamp}`; + let holder; + let release; + let anchors; + let failedAt; + + const holds = async (name) => (await holder.getInstalledApps()).data?.some((a) => a.name === name) ?? null; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 3, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await Promise.all([appName, otherName].map((name) => pushImage(name, 'v1'))); + const app = await buildSeedableApp({ env, name: appName, compose: [component(appName)] }); + const other = await buildSeedableApp({ env, name: otherName, compose: [component(otherName)] }); + await installOnNodes(env, app, [HOLDER]); + await installOnNodes(env, other, [HOLDER]); + await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)) + && Boolean(await getAppContainerId(holder.container, otherName, otherName)), { + timeout: 120000, interval: 2000, label: 'both apps run before the redeploy', + }); + + const auth = await authenticate(holder.url, appOwnerKey()); + release = await failCommands(HOLDER + 1, 'localzelapps', 'zelappsinformation', { commands: ['findAndModify'], times: REFUSED }); + step(`hard redeploy with the row delete refused ${REFUSED} times`); + const body = await holder.redeployApp(appName, auth.zelidauth, { force: true }); + if (/Unauthorized/i.test(body)) throw new Error(`hard redeploy refused: ${body.slice(0, 400)}`); + anchors = env.clients.map((client) => client?.getLastEventId()); + failedAt = holder.getLastEventId(); + step('redeploy returned'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('no longer holds the app once its removal finishes', async function () { + this.timeout(240000); + await waitFor(async () => (await holds(appName)) === false, { + timeout: 210000, interval: 2000, label: 'the node no longer holds the app', + }); + }); + + it('had its deletes of the app\'s row refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database refused none of the node\'s row deletes, so nothing here was tested').to.be.at.least(1); + }); + + it('never announces the app once its removal has stopped part way', function () { + const announcements = holder.getEventBuffer().filter((e) => e.id > failedAt && e.event === 'app:running'); + expect( + announcements.some((e) => e.data.apps?.some((a) => a.name === otherName)), + 'the node announced nothing while the app was held, so nothing here was tested', + ).to.equal(true); + expect( + announcements.filter((e) => e.data.apps?.some((a) => a.name === appName)).length, + 'the node announced an app it had part removed', + ).to.equal(0); + }); + + it('tells the network it no longer holds the app', async function () { + this.timeout(90000); + const peers = env.clients.filter((client, i) => client && i !== HOLDER); + const heard = await Promise.any(peers.map((peer) => peer.waitForEvent( + 'network:appremoved', + (d) => d.name === appName && d.ip.startsWith(holder.ip), + 60000, + { afterId: anchors[env.clients.indexOf(peer)] }, + ))).catch(() => null); + expect(heard, 'no node was told the app had gone').to.not.equal(null); + }); +}); From ede377d2c5adc165ac8290ee0e30108b5a942906 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 15:00:10 +0100 Subject: [PATCH 129/369] test(harness): failCommands fails a command as a timed-out call, which nothing retries The failpoint answers MaxTimeMSExpired: the driver neither retries the command nor clears the node's connection pool, so only the named commands fail and `times` counts calls. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/mongo-control.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js index abe213131c..cd0afae474 100644 --- a/test-infra/runner/framework/mongo-control.js +++ b/test-infra/runner/framework/mongo-control.js @@ -55,17 +55,18 @@ export async function releaseCommands() { }); } -// The given commands on one collection of one node fail, as a call the database could -// not complete does, and every other node's go through: each node keeps its data in -// databases of its own (dbClient's prefix). `times` fails that many and then stops; left -// out, every one fails until released. Resolves to a release that turns the failure off -// and resolves to how many commands it failed. +// The given commands on one collection of one node fail, as a call that ran out of time +// does (MaxTimeMSExpired): the driver neither retries it nor drops its other +// connections, so nothing but the named commands fails. Every other node's go through: +// each node keeps its data in databases of its own (dbClient's prefix). `times` fails +// that many and then stops; left out, every one fails until released. Resolves to a +// release that turns the failure off and resolves to how many commands it failed. export async function failCommands(nodeNum, database, collection, { commands, times } = {}) { const namespace = `node${String(nodeNum).padStart(2, '0')}_${database}.${collection}`; await withAdmin((admin) => admin.command({ configureFailPoint: 'failCommand', mode: times === undefined ? 'alwaysOn' : { times }, - data: { failCommands: commands, namespace, errorCode: 91 }, + data: { failCommands: commands, namespace, errorCode: 50 }, })); return async function release() { // turning a failpoint off answers how many times it fired From 55a6ce4f15bb847a11a455436b87fbbfac177598 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 15:22:12 +0100 Subject: [PATCH 130/369] fix(apps): a component removal that stops part way removes the whole app An app missing a component cannot run, so a failure once a component's teardown has begun hands the whole app to finishRemoval: held as departing, removed whole, and the network told once it is gone. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appUninstaller.js | 12 ++++++------ tests/unit/appUninstaller.test.js | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 68544c64ab..292461646e 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -910,8 +910,8 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // the same finally, and a refused duplicate must not release the lock the removal // actually running is holding. let acquired = false; - // Set once the teardown of a whole app has begun: a failure from there on leaves - // the app part removed. + // Set once a teardown has begun: a failure from there on leaves the app part + // removed, and an app missing a component cannot run, so it is the whole app. let tornDownApp = null; let announced = false; try { @@ -1016,7 +1016,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen appSpecifications = await checkAndDecryptAppSpecs(appSpecifications); appSpecifications = specificationFormatter(appSpecifications); - if (!isComponent) tornDownApp = appName; + tornDownApp = appName; if (appSpecifications.version >= 4 && !isComponent) { // it is a composed application // eslint-disable-next-line no-restricted-syntax @@ -1179,9 +1179,9 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen } } // A specification that cannot be found anywhere means the node is not holding - // this app, which answers a caller asking for it to be gone. A whole app whose - // teardown had begun is finished in the background, and the network is told - // once it is gone. Any other error leaves the app as it was. + // this app, which answers a caller asking for it to be gone. An app whose + // teardown had begun, of the whole app or of one component, is removed whole in + // the background, and the network is told once it is gone. Any other error leaves the app as it was. if (error instanceof AppNotFoundError) return RemovalOutcome.NOT_INSTALLED; if (tornDownApp) finishRemoval(tornDownApp, force, !announced); return RemovalOutcome.FAILED; diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 0885e8821c..6ea0de208a 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -767,6 +767,22 @@ describe('appUninstaller tests', () => { sinon.assert.calledOnce(getLocalSocketAddressStub); }); + it('removes the whole app when the removal of one of its components stops part way', async () => { + let heldWhileFinishing = null; + const deleteRow = sinon.stub().callsFake(async () => { + heldWhileFinishing = globalStateStub.departingApps.has('testapp'); + }); + const uninstaller = buildUninstaller(composedSpec, {}, { findOneAndDeleteInDatabase: deleteRow }); + runtimeStateStub.remove.onFirstCall().rejects(new Error('connection timed out')); + + const outcome = await uninstaller.removeAppLocally('comp1_testapp', res, true); + await settle(() => deleteRow.called && !globalStateStub.departingApps.has('testapp')); + + expect(outcome).to.equal('failed'); + sinon.assert.calledOnce(deleteRow); + expect(heldWhileFinishing, 'the app was not held while it was removed whole').to.be.true; + }); + it('fails the removal, before the row, when the runtime state cannot be removed, and finishes it', async () => { const deleteRow = sinon.stub().resolves(); const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); From 5d75714c97240f8389957487483b6c513497c253 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 15:56:38 +0100 Subject: [PATCH 131/369] fix(apps): an app whose removal is pending takes no install A part-finished removal is run again until it finishes, and that run removes whatever holds the app's name. registerAppLocally and softRegisterAppLocally answer BUSY for an app held as departing, and the reinstall pass leaves it for a later pass. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 18 +++++++++++++ .../src/services/appLifecycle/appInstaller.js | 11 ++++++++ tests/unit/advancedWorkflows.test.js | 25 +++++++++++++++++++ tests/unit/appInstaller.test.js | 13 ++++++++++ 4 files changed, 67 insertions(+) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 6e4825dc15..31d3009626 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -1035,6 +1035,17 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { } return InstallOutcome.BUSY; } + // A pending removal of the app removes whatever is installed in its place, so + // the app takes no install until that removal has finished. + if (globalState.departingApps.has(appSpecs.name)) { + const rStatus = messageHelper.createErrorMessage(`Flux App ${appSpecs.name} is being removed`); + log.error(rStatus); + if (res) { + res.write(serviceHelper.ensureString(rStatus)); + if (res.flush) res.flush(); + } + return InstallOutcome.BUSY; + } globalState.installationInProgress = true; acquired = true; const tier = await generalService.nodeTier().catch((error) => log.error(error)); @@ -4231,6 +4242,13 @@ async function reinstallOldApplications() { const appsInstalled = installedAppsRes.data; // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled) { + // An app whose removal is pending is not rebuilt: that removal would take + // the rebuilt app too. A later pass finds it again if it is still installed. + if (globalState.departingApps.has(installedApp.name)) { + log.info(`reinstallOldApplications - ${installedApp.name} is being removed, leaving it for a later pass`); + // eslint-disable-next-line no-continue + continue; + } // get current app specifications for the app name // if match found. Check if hash found. // if same, do nothing. if different remove and install. diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 6da5b03dab..09fa9052ba 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -370,6 +370,17 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f } return InstallOutcome.BUSY; } + // A pending removal of the app removes whatever is installed in its place, so + // the app takes no install until that removal has finished. + if (globalState.departingApps.has(appSpecs.name)) { + const rStatus = messageHelper.createWarningMessage(`Flux App ${appSpecs.name} is being removed. Installation not possible`); + log.error(rStatus); + if (res) { + res.write(serviceHelper.ensureString(rStatus)); + if (res.flush) res.flush(); + } + return InstallOutcome.BUSY; + } globalState.installationInProgress = true; acquired = true; // A test install holds nothing: it writes the app's row like any other install diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 4cd28d2c2c..0ae676dd96 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -132,6 +132,19 @@ describe('advancedWorkflows tests', () => { expect(nodeTier.called, 'the refusal is decided before any work is done').to.be.false; expect(globalState.installationInProgress, 'a refusal released someone else\'s hold').to.be.true; }); + + it('refuses an app whose removal is pending', async () => { + const nodeTier = sinon.stub(generalService, 'nodeTier').resolves('cumulus'); + globalState.departingApps.enter(appSpec.name); + try { + const result = await advancedWorkflows.softRegisterAppLocally(appSpec, false, makeRes()); + + expect(result).to.equal(InstallOutcome.BUSY); + expect(nodeTier.called, 'the refusal is decided before any work is done').to.be.false; + } finally { + globalState.departingApps.leave(appSpec.name); + } + }); }); describe('setRemovalInProgress and getRemovalInProgress tests', () => { @@ -7350,6 +7363,18 @@ describe('advancedWorkflows tests', () => { // redeploys nothing. Nothing on that branch judges what it wrote: the installer // judges what it installs, and this installs nothing, so an owner the network // refuses holds the application until something unrelated sweeps the node. + it('leaves an app whose removal is pending to a later pass', async () => { + const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + globalState.departingApps.enter(installedApp.name); + try { + await advancedWorkflows.reinstallOldApplications(); + } finally { + globalState.departingApps.leave(installedApp.name); + } + + expect(softUninstallComponent.called, 'rebuilt an app its pending removal would take again').to.be.false; + }); + it('asks the sweep about an application whose record it rewrote without redeploying', async () => { const requestComplianceSweep = sinon.stub(imageManager, 'requestComplianceSweep').resolves(); dbHelper.findOneInDatabase.resolves({ diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 6014a1306e..72f9188e45 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -1000,6 +1000,19 @@ describe('appInstaller tests', () => { expect(globalStateStub.installationInProgress, 'a refusal released someone else\'s hold').to.be.true; }); + it('refuses an app whose removal is pending, taking no hold', async () => { + const res = { write: sinon.stub(), end: sinon.stub() }; + globalStateStub.departingApps.enter(appSpec.name); + try { + const result = await appInstaller.registerAppLocally(appSpec, false, res); + + expect(result).to.equal(InstallOutcome.BUSY); + expect(globalStateStub.installationInProgress, 'the refusal took the install hold').to.be.false; + } finally { + globalStateStub.departingApps.leave(appSpec.name); + } + }); + it('should return false if node tier does not return anything', async () => { const appInstallerWithNodeTier = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { config: configStub, From e6ec5246b75b3d7423871df42d8d0e2cf17945c8 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 15:57:38 +0100 Subject: [PATCH 132/369] test(harness): 3309, an app whose removal is pending takes no install A hard redeploy whose row delete is refused leaves the app held; a second hard redeploy, sent before the removal runs again, removes it and is refused the install, and the node never installs the app. Co-Authored-By: Claude Opus 5.5 --- ...removal-that-stops-part-way-is-finished.js | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js b/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js index a879181a9f..4d38e57e99 100644 --- a/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js +++ b/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js @@ -7,6 +7,10 @@ * The node's deletes of the row are refused REFUSED times: the redeploy's removal and * the next runs of it, so the app is held across at least one announcement. A second * app on the node is in every announcement, which shows announcements went out. + * + * While the app is held, nothing installs it: the removal that finishes it would take + * whatever held its name. A second hard redeploy, sent before the removal runs again, + * removes the app and is refused the install. */ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; @@ -128,3 +132,62 @@ describe('a removal that stops part way is finished', function () { expect(heard, 'no node was told the app had gone').to.not.equal(null); }); }); + +describe('an app whose removal is pending takes no install', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2eheldnoinstall${Date.now()}`; + let holder; + let release; + let afterFailure; + let failedDeletes; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 3, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ env, name: appName, compose: [component(appName)] }); + await installOnNodes(env, app, [HOLDER]); + await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)), { + timeout: 120000, interval: 2000, label: 'the app runs before the redeploys', + }); + + const auth = await authenticate(holder.url, appOwnerKey()); + release = await failCommands(HOLDER + 1, 'localzelapps', 'zelappsinformation', { commands: ['findAndModify'], times: 1 }); + step('hard redeploy with the row delete refused once'); + const first = await holder.redeployApp(appName, auth.zelidauth, { force: true }); + if (/Unauthorized/i.test(first)) throw new Error(`hard redeploy refused: ${first.slice(0, 400)}`); + failedDeletes = await release(); + release = null; + afterFailure = holder.getLastEventId(); + step('second hard redeploy while the app is held'); + await holder.redeployApp(appName, auth.zelidauth, { force: true }); + step('second redeploy returned'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('had its delete of the app\'s row refused by the database', function () { + expect(failedDeletes, 'the database refused none of the node\'s row deletes, so nothing here was tested').to.equal(1); + }); + + it('no longer holds the app once its removal finishes', async function () { + this.timeout(120000); + await waitFor(async () => (await holder.getInstalledApps()).data?.some((a) => a.name === appName) === false, { + timeout: 90000, interval: 2000, label: 'the node no longer holds the app', + }); + }); + + it('never installs the app while its removal is pending', function () { + const installed = holder.getEventBuffer().filter((e) => e.id > afterFailure && e.event === 'app:installed' + && e.data.name === appName); + expect(installed.length, 'the node installed an app its pending removal then took').to.equal(0); + }); +}); From 84b37505f18c464a932fd3c198b335b2962a0cc5 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 18:32:11 +0100 Subject: [PATCH 133/369] fix(apps): the election says which app it leaves out when a stop lock cannot be read Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/advancedWorkflows.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 31d3009626..184d5454fa 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -5132,7 +5132,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, members = await electedMembers(installedApp); } catch (error) { // An unread stop lock is not a lifted one: not a candidate this pass. - log.warn(`masterSlaveApps: ${installedApp.name} stop locks could not be read - excluded from primary election this pass: ${error.message}`); + log.warn(`masterSlaveApps: ${installedApp.name}: a stop lock could not be read - excluded from primary election this pass: ${error.message}`); // eslint-disable-next-line no-continue continue; } From 17175c872e9c75f489aaafb4fd058ffeb18e38be Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 07:38:09 +0100 Subject: [PATCH 134/369] fix(upnp): a test mapping the router has not removed fails the availability check The availability check maps its test port over TCP alone: its test server is HTTP. A pass that finds the router has not removed the last test mapping asks for the removal at once and, while it is refused, makes no new test mapping and counts as a failed check, so a router that keeps refusing puts the node in DOS, and the pass after it removes the mapping tests and releases it. mapUpnpPort takes the protocols to map, TCP and then UDP unless given. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/availabilityChecker.js | 16 ++-- ZelBack/src/services/upnpService.js | 77 +++++++++++++------ tests/unit/availabilityChecker.test.js | 50 +++++++++++- tests/unit/upnpService.test.js | 38 +++++++++ 4 files changed, 147 insertions(+), 34 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index e74a99d952..194b2807c4 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -237,11 +237,13 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } } - // A router that refused to remove the last test mapping is not given - // another until it has. - if (isUpnp && upnpService.testRemovalPending()) { - log.info('checkMyAppsAvailability - a test mapping is still to be removed from the router; no new one until it is'); - await serviceHelper.delay(timeouts.error); + // A test the router cannot remove the mapping of has not completed: the + // pass asks for the removal again, and while it is refused makes no new + // test mapping and counts as a failed check. + if (isUpnp && upnpService.testRemovalPending() && !await upnpService.askPendingTestRemovals()) { + log.info('checkMyAppsAvailability - the router has not removed the last test mapping; no new one until it has'); + countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); + await serviceHelper.delay(dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -276,13 +278,13 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } if (isUpnp) { - const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App'); + // the test server is HTTP: TCP is all it needs + const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); routerRefused = routerRefusedSinceLastPass(dosState); if (routerRefused) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); if (!upnpMapResult.mapped) { log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} failed to create UPnP mapping: ${upnpMapResult.reason}`); setNextPort(); - // the router may have kept the TCP half await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipHttpServer: true, diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 0b8beaeed1..8f0b4092da 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -399,26 +399,31 @@ async function setupUPNP(apiport = config.server.apiport) { } /** - * Maps a port to this node over TCP and then UDP. + * Maps a port to this node over each of `protocols` in turn, TCP and then UDP + * unless given. * - * On a failure the port is no longer held here; a TCP half the router took is + * On a failure the port is no longer held here; a protocol the router took is * still recorded, for removeMapUpnpPort to remove. * @param {number} port Port number. * @param {string} description Port description. + * @param {{protocols?: string[]}} [options] * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} - * the first refusal, as mapProtocol gives it; mapped when both protocols are + * the first refusal, as mapProtocol gives it; mapped when every protocol is */ -async function mapUpnpPort(port, description) { +async function mapUpnpPort(port, description, { protocols = ['TCP', 'UDP'] } = {}) { // held from before the router has it: the cleanup pass may list it the // moment it is there mappedHere.add(+port); dropPendingRemovals(+port); - let result = await mapProtocol(+port, 'TCP', description); - recordMapped(+port, 'TCP', result.mapped); - if (result.mapped) { - await serviceHelper.delay(500); - result = await mapProtocol(+port, 'UDP', description); - recordMapped(+port, 'UDP', result.mapped); + let result = MAPPED; + // eslint-disable-next-line no-restricted-syntax + for (const [i, protocol] of protocols.entries()) { + // eslint-disable-next-line no-await-in-loop + if (i > 0) await serviceHelper.delay(500); + // eslint-disable-next-line no-await-in-loop + result = await mapProtocol(+port, protocol, description); + recordMapped(+port, protocol, result.mapped); + if (!result.mapped) break; } if (!result.mapped) { log.error(`UPnP - ${result.reason}`); @@ -461,8 +466,29 @@ function dropPendingRemovals(port) { /** * Asks the router once more to remove a pending removal's mapping, in turn - * behind any other pending removal, and waits longer before the next ask if it - * refuses again. + * behind any other pending removal, and schedules the next ask if it refuses + * again. + * @param {object} pending + * @returns {Promise} settles once the router has been asked + */ +function askPendingRemoval(pending) { + pendingRemovalTurn = pendingRemovalTurn.then(async () => { + const key = `${pending.port}/${pending.protocol}`; + if (pendingRemovals.get(key) !== pending) return; + if (await removeProtocol(pending.port, pending.protocol)) { + pendingRemovals.delete(key); + log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); + return; + } + // mapped again while the router was being asked: in use, not pending + if (pendingRemovals.get(key) === pending) schedulePendingRemoval(pending); + }); + return pendingRemovalTurn; +} + +/** + * Asks the router again to remove a pending removal's mapping after a wait + * that doubles with each refusal, up to config.upnp.removalRetryMaxMs. * @param {object} pending */ function schedulePendingRemoval(pending) { @@ -471,19 +497,7 @@ function schedulePendingRemoval(pending) { // eslint-disable-next-line no-param-reassign pending.refusals += 1; // eslint-disable-next-line no-param-reassign - pending.timer = setTimeout(() => { - pendingRemovalTurn = pendingRemovalTurn.then(async () => { - const key = `${pending.port}/${pending.protocol}`; - if (pendingRemovals.get(key) !== pending) return; - if (await removeProtocol(pending.port, pending.protocol)) { - pendingRemovals.delete(key); - log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); - return; - } - // mapped again while the router was being asked: in use, not pending - if (pendingRemovals.get(key) === pending) schedulePendingRemoval(pending); - }); - }, waitMs); + pending.timer = setTimeout(() => askPendingRemoval(pending), waitMs); } /** @@ -496,6 +510,18 @@ function testRemovalPending() { return [...pendingRemovals.values()].some((pending) => isTestMappingDescription(pending.description)); } +/** + * Asks the router now to remove every pending removal of this node's test + * mappings, without waiting for their next scheduled ask. + * @returns {Promise} whether none is pending any longer + */ +async function askPendingTestRemovals() { + const tests = [...pendingRemovals.values()].filter((pending) => isTestMappingDescription(pending.description)); + tests.forEach((pending) => clearTimeout(pending.timer)); + await Promise.all(tests.map(askPendingRemoval)); + return !testRemovalPending(); +} + /** * Removes the protocols of a port that this node mapped, each on its own: a * refusal of one leaves the other to go. A router removes a mapping by its @@ -884,6 +910,7 @@ module.exports = { mappedProtocols, pendingRemovals, testRemovalPending, + askPendingTestRemovals, CLEANUP_SLOT_MS, BOOT_LISTING_STAGGER_MS, mapPortApi, diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index d97a2f67ca..8b3bdc1f09 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -362,6 +362,7 @@ describe('availabilityChecker tests', () => { sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); sinon.stub(upnpService, 'routerRefusals').callsFake(() => refusals); sinon.stub(upnpService, 'testRemovalPending').returns(false); + sinon.stub(upnpService, 'askPendingTestRemovals').resolves(false); mapUpnpPortStub = sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); removeMapUpnpPortStub = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { @@ -405,15 +406,60 @@ describe('availabilityChecker tests', () => { expect(mockDosState.portFailureScore ?? 0).to.equal(0); }); - it('makes no test mapping, and opens no port, while the router has yet to remove the last one', async () => { + it('maps the port under test over TCP alone', async () => { + await pass(); + + sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); + }); + + it('makes no test mapping, opens no port, and counts a failed check while the router refuses to remove the last one', async () => { upnpService.testRemovalPending.returns(true); await pass(); + sinon.assert.calledOnce(upnpService.askPendingTestRemovals); sinon.assert.notCalled(mapUpnpPortStub); sinon.assert.notCalled(fluxNetworkHelper.allowTestPort); sinon.assert.notCalled(portTestStub); - expect(mockDosState.portFailureScore ?? 0, 'counts nothing').to.equal(0); + expect(mockDosState.portFailureScore).to.equal(4); + sinon.assert.calledWith(delayStub, 60_000); + }); + + it('holds the node out of service once the router has refused to remove the last test mapping for as many passes as the threshold takes', async () => { + upnpService.testRemovalPending.returns(true); + + // eslint-disable-next-line no-restricted-syntax + for (let i = 0; i < 25; i += 1) { + // eslint-disable-next-line no-await-in-loop + await pass(); + } + + expect(mockDosState.portFailureScore).to.equal(100); + expect(mockDosState.dosMessage).to.equal(UPNP_DOS); + expect(mockDosState.dosStateValue).to.equal(100); + sinon.assert.calledWith(delayStub.lastCall, 300_000); + }); + + it('tests, and ends its DOS, on the pass whose ask the router removes the last test mapping on', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + upnpService.testRemovalPending.returns(true); + upnpService.askPendingTestRemovals.resolves(true); + + await pass(); + + sinon.assert.calledOnce(mapUpnpPortStub); + sinon.assert.calledOnce(portTestStub); + expect(mockDosState.dosMessage).to.equal(null); + expect(mockDosState.dosStateValue).to.equal(0); + expect(mockDosState.portFailureScore).to.equal(0); + }); + + it('asks the router for no removal when none is pending', async () => { + await pass(); + + sinon.assert.notCalled(upnpService.askPendingTestRemovals); }); it('removes what the router kept of a test mapping it refused', async () => { diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 22ad2dd276..0920ccd200 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -819,6 +819,17 @@ describe('upnpService tests', () => { }); }); + it('should map only the protocols it is given', async () => { + upnpService.mappedProtocols.clear(); + const result = await upnpService.mapUpnpPort(123, 'Flux_Test_App', { protocols: ['TCP'] }); + + expect(result.mapped).to.equal(true); + sinon.assert.calledOnceWithExactly(createMappingStub, { + public: 123, private: 123, ttl: 0, protocol: 'TCP', description: 'Flux_Test_App', + }); + expect([...upnpService.mappedProtocols.get(123)]).to.deep.equal(['TCP']); + }); + it('should not ask for UDP when TCP is refused', async () => { createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); @@ -1252,6 +1263,33 @@ describe('upnpService tests', () => { expect(most, 'never two at once').to.equal(1); }); + it('should ask now for a test mapping\'s pending removal, and say whether it has gone', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + expect(await upnpService.askPendingTestRemovals(), 'refused again').to.equal(false); + expect(removals().length, 'asked at once, not after the wait').to.equal(2); + expect(upnpService.testRemovalPending()).to.equal(true); + + expect(await upnpService.askPendingTestRemovals(), 'removed').to.equal(true); + expect(removals().length).to.equal(3); + expect(upnpService.pendingRemovals.size).to.equal(0); + await clock.tickAsync(MAX_MS); + expect(removals().length, 'no scheduled ask left behind').to.equal(3); + }); + + it('should ask now for no app mapping\'s pending removal', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + expect(await upnpService.askPendingTestRemovals()).to.equal(true); + expect(removals().length).to.equal(1); + expect(upnpService.pendingRemovals.has('123/TCP')).to.equal(true); + }); + it('should hold back new test mappings while a test mapping\'s removal is pending, and only then', async () => { await mapTcpOnly(123, 'Flux_App_web'); removeMappingStub.onCall(0).rejects(refused()); From 41c530ba10eadee1ff921ce2dfa8e7f3fdd54ebe Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 07:38:12 +0100 Subject: [PATCH 135/369] fix(upnp): a refused removal of one protocol keeps the other's pending A removal the router refuses replaces only that protocol's pending removal, so when it refuses both protocols of a port both are asked again, and a second removal of a port with nothing mapped since keeps the first's. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 5 +++-- tests/unit/upnpService.test.js | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 8f0b4092da..25dd0a7995 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -547,8 +547,9 @@ async function removeMapUpnpPort(port, description = '') { const pending = { port: +port, protocol, description, refusals: 0, timer: null, }; - dropPendingRemovals(+port); - pendingRemovals.set(`${+port}/${protocol}`, pending); + const key = `${+port}/${protocol}`; + clearTimeout(pendingRemovals.get(key)?.timer); + pendingRemovals.set(key, pending); schedulePendingRemoval(pending); } // eslint-disable-next-line no-await-in-loop diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 0920ccd200..b79080e79c 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -1263,6 +1263,30 @@ describe('upnpService tests', () => { expect(most, 'never two at once').to.equal(1); }); + it('should keep asking for both protocols of a port when the router refuses to remove either', async () => { + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + expect([...upnpService.pendingRemovals.keys()].sort()).to.deep.equal(['123/TCP', '123/UDP']); + await clock.tickAsync(FIRST_MS); + expect(removals().slice(2).sort()).to.deep.equal(['123/TCP', '123/UDP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should keep a pending removal when the port is removed again with nothing mapped since', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + expect(await upnpService.removeMapUpnpPort(123, 'Flux_App_web')).to.equal(true); + await clock.tickAsync(FIRST_MS); + + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + it('should ask now for a test mapping\'s pending removal, and say whether it has gone', async () => { await mapTcpOnly(123, 'Flux_Test_App'); removeMappingStub.onCall(0).rejects(refused()); From 12eb2fa27e130ad9baa59334220fc654d8c0130d Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 08:10:00 +0100 Subject: [PATCH 136/369] fix(upnp): a pending removal of any mapping fails the availability check, and UPnP wakes it upnpService alone asks the router again for a removal it refused, the wait doubling from 30 s to at most 60 s, and says when a removal becomes pending with none before ('pending') and when the last goes ('clear'). The availability check reads that state: while any removal is pending it makes no test mapping and counts each pass as a failed check, so a router that keeps refusing puts the node in DOS. Either signal starts its next pass at once, so a refusal is acted on without waiting out a passing test's hour, and the pass after the router removes the mapping tests and releases the node. A pass that leaves a removal pending waits no longer than an error's wait. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 2 +- .../appMonitoring/availabilityChecker.js | 39 +++++-- ZelBack/src/services/upnpService.js | 70 ++++++------ tests/unit/availabilityChecker.test.js | 48 ++++---- tests/unit/globalconfig/default.js | 2 +- tests/unit/upnpService.test.js | 103 ++++++++++-------- 6 files changed, 158 insertions(+), 106 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index ba45df6173..12a34753ed 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -110,7 +110,7 @@ module.exports = { // a removal of this node's own mapping the router refuses is asked again // after this long, the wait doubling up to removalRetryMaxMs removalRetryFirstMs: 30 * 1000, - removalRetryMaxMs: 30 * 60 * 1000, + removalRetryMaxMs: 60 * 1000, }, benchmark: { host: '127.0.0.1', diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index 194b2807c4..fce3189c30 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -97,6 +97,30 @@ function releaseOwnDos(dosState) { dosState.dosStateValue = 0; } +let wakeNextPass = null; + +/** + * Waits `ms` before the next pass, or less: a UPnP removal becoming pending, or + * the last one going, starts the next pass at once. + * @param {number} ms + * @returns {Promise} + */ +async function waitForNextPass(ms) { + const woken = new Promise((resolve) => { wakeNextPass = resolve; }); + await Promise.race([serviceHelper.delay(ms), woken]); + wakeNextPass = null; +} + +/** + * Starts the next pass now, if the check is waiting for it. + */ +function wakeCheck() { + if (wakeNextPass) wakeNextPass(); +} + +upnpService.removalState.on('pending', wakeCheck); +upnpService.removalState.on('clear', wakeCheck); + /** * Whether the router has refused a mapping for a reason of its own - a full * table, no answer - since this check last asked: any mapping, an install's, a @@ -237,13 +261,12 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } } - // A test the router cannot remove the mapping of has not completed: the - // pass asks for the removal again, and while it is refused makes no new - // test mapping and counts as a failed check. - if (isUpnp && upnpService.testRemovalPending() && !await upnpService.askPendingTestRemovals()) { - log.info('checkMyAppsAvailability - the router has not removed the last test mapping; no new one until it has'); + // A router that has yet to remove a mapping of this node fails the check, + // and is given no test mapping until it has. + if (isUpnp && upnpService.removalPending()) { + log.info('checkMyAppsAvailability - the router has yet to remove a mapping of this node; no test mapping until it has'); countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); - await serviceHelper.delay(dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); + await waitForNextPass(dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -436,7 +459,9 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin log.error(`checkMyAppsAvailability - Count: ${portsNotWorking.size}. portsNotWorking: ${JSON.stringify(Array.from(portsNotWorking))}`); } - await serviceHelper.delay(waitMs); + // a removal refused during this pass gave no wake: the check was not waiting + if (isUpnp && upnpService.removalPending()) waitMs = Math.min(waitMs, timeouts.error); + await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp }); diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 25dd0a7995..6933f6c70d 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -1,3 +1,4 @@ +const { EventEmitter } = require('node:events'); const config = require('config'); const natUpnp = require('@runonflux/nat-upnp'); const serviceHelper = require('./serviceHelper'); @@ -54,6 +55,34 @@ const mappedProtocols = new Map(); const pendingRemovals = new Map(); let pendingRemovalTurn = Promise.resolve(); +// Emits 'pending' when a removal becomes pending with none pending before, and +// 'clear' when the last pending removal goes. +const removalState = new EventEmitter(); + +/** + * Makes a removal pending under `key`, in place of any pending there. + * @param {string} key `port/protocol` + * @param {object} pending + */ +function setPendingRemoval(key, pending) { + const wasClear = pendingRemovals.size === 0; + clearTimeout(pendingRemovals.get(key)?.timer); + pendingRemovals.set(key, pending); + if (wasClear) removalState.emit('pending'); +} + +/** + * Ends the pending removal under `key`, if there is one. + * @param {string} key `port/protocol` + */ +function deletePendingRemoval(key) { + const pending = pendingRemovals.get(key); + if (!pending) return; + clearTimeout(pending.timer); + pendingRemovals.delete(key); + if (pendingRemovals.size === 0) removalState.emit('clear'); +} + /** * Records whether this node's latest map of one protocol of a port left the * router mapping it here. @@ -456,12 +485,7 @@ async function removeProtocol(port, protocol) { * @param {number} port */ function dropPendingRemovals(port) { - ['TCP', 'UDP'].forEach((protocol) => { - const pending = pendingRemovals.get(`${port}/${protocol}`); - if (!pending) return; - clearTimeout(pending.timer); - pendingRemovals.delete(`${port}/${protocol}`); - }); + ['TCP', 'UDP'].forEach((protocol) => deletePendingRemoval(`${port}/${protocol}`)); } /** @@ -469,21 +493,19 @@ function dropPendingRemovals(port) { * behind any other pending removal, and schedules the next ask if it refuses * again. * @param {object} pending - * @returns {Promise} settles once the router has been asked */ function askPendingRemoval(pending) { pendingRemovalTurn = pendingRemovalTurn.then(async () => { const key = `${pending.port}/${pending.protocol}`; if (pendingRemovals.get(key) !== pending) return; if (await removeProtocol(pending.port, pending.protocol)) { - pendingRemovals.delete(key); + deletePendingRemoval(key); log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); return; } // mapped again while the router was being asked: in use, not pending if (pendingRemovals.get(key) === pending) schedulePendingRemoval(pending); }); - return pendingRemovalTurn; } /** @@ -501,25 +523,11 @@ function schedulePendingRemoval(pending) { } /** - * Whether a removal of one of this node's test mappings is still pending: no - * new test mapping is made until it has gone, so a router refusing removals - * gathers no more of them. + * Whether the router has yet to remove a mapping of this node it was asked to. * @returns {boolean} */ -function testRemovalPending() { - return [...pendingRemovals.values()].some((pending) => isTestMappingDescription(pending.description)); -} - -/** - * Asks the router now to remove every pending removal of this node's test - * mappings, without waiting for their next scheduled ask. - * @returns {Promise} whether none is pending any longer - */ -async function askPendingTestRemovals() { - const tests = [...pendingRemovals.values()].filter((pending) => isTestMappingDescription(pending.description)); - tests.forEach((pending) => clearTimeout(pending.timer)); - await Promise.all(tests.map(askPendingRemoval)); - return !testRemovalPending(); +function removalPending() { + return pendingRemovals.size > 0; } /** @@ -529,7 +537,7 @@ async function askPendingTestRemovals() { * not take - one another machine holds - is never asked for. A removal the * router refuses is pending, and asked again until it goes. * @param {number} port Port number. - * @param {string} [description] the mapping's description, for the log and testRemovalPending + * @param {string} [description] the mapping's description, for the log * @returns {Promise} whether the router removed every one of them now */ async function removeMapUpnpPort(port, description = '') { @@ -547,9 +555,7 @@ async function removeMapUpnpPort(port, description = '') { const pending = { port: +port, protocol, description, refusals: 0, timer: null, }; - const key = `${+port}/${protocol}`; - clearTimeout(pendingRemovals.get(key)?.timer); - pendingRemovals.set(key, pending); + setPendingRemoval(`${+port}/${protocol}`, pending); schedulePendingRemoval(pending); } // eslint-disable-next-line no-await-in-loop @@ -910,8 +916,8 @@ module.exports = { mappedHere, mappedProtocols, pendingRemovals, - testRemovalPending, - askPendingTestRemovals, + removalPending, + removalState, CLEANUP_SLOT_MS, BOOT_LISTING_STAGGER_MS, mapPortApi, diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 8b3bdc1f09..8ca19e9e7a 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -361,8 +361,7 @@ describe('availabilityChecker tests', () => { sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); sinon.stub(upnpService, 'routerRefusals').callsFake(() => refusals); - sinon.stub(upnpService, 'testRemovalPending').returns(false); - sinon.stub(upnpService, 'askPendingTestRemovals').resolves(false); + sinon.stub(upnpService, 'removalPending').returns(false); mapUpnpPortStub = sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); removeMapUpnpPortStub = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { @@ -412,12 +411,11 @@ describe('availabilityChecker tests', () => { sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); }); - it('makes no test mapping, opens no port, and counts a failed check while the router refuses to remove the last one', async () => { - upnpService.testRemovalPending.returns(true); + it('makes no test mapping, opens no port, and counts a failed check while the router has yet to remove a mapping of this node', async () => { + upnpService.removalPending.returns(true); await pass(); - sinon.assert.calledOnce(upnpService.askPendingTestRemovals); sinon.assert.notCalled(mapUpnpPortStub); sinon.assert.notCalled(fluxNetworkHelper.allowTestPort); sinon.assert.notCalled(portTestStub); @@ -425,8 +423,8 @@ describe('availabilityChecker tests', () => { sinon.assert.calledWith(delayStub, 60_000); }); - it('holds the node out of service once the router has refused to remove the last test mapping for as many passes as the threshold takes', async () => { - upnpService.testRemovalPending.returns(true); + it('holds the node out of service once the router has yet to remove a mapping for as many passes as the threshold takes', async () => { + upnpService.removalPending.returns(true); // eslint-disable-next-line no-restricted-syntax for (let i = 0; i < 25; i += 1) { @@ -440,28 +438,40 @@ describe('availabilityChecker tests', () => { sinon.assert.calledWith(delayStub.lastCall, 300_000); }); - it('tests, and ends its DOS, on the pass whose ask the router removes the last test mapping on', async () => { - mockDosState.portFailureScore = 100; - mockDosState.dosStateValue = 100; - mockDosState.dosMessage = UPNP_DOS; - upnpService.testRemovalPending.returns(true); - upnpService.askPendingTestRemovals.resolves(true); + it('waits no longer than an error\'s wait after a pass that left a removal pending', async () => { + upnpService.removalPending.onFirstCall().returns(false).returns(true); await pass(); - sinon.assert.calledOnce(mapUpnpPortStub); sinon.assert.calledOnce(portTestStub); - expect(mockDosState.dosMessage).to.equal(null); - expect(mockDosState.dosStateValue).to.equal(0); - expect(mockDosState.portFailureScore).to.equal(0); + sinon.assert.calledWith(delayStub.lastCall, 60_000); }); - it('asks the router for no removal when none is pending', async () => { + it('waits the whole wait after a passing test that left nothing pending', async () => { await pass(); - sinon.assert.notCalled(upnpService.askPendingTestRemovals); + sinon.assert.calledWith(delayStub.lastCall, 3_600_000); }); + // eslint-disable-next-line no-restricted-syntax + for (const event of ['pending', 'clear']) { + it(`starts its next pass at once when UPnP says its removals are ${event}`, async () => { + delayStub.returns(new Promise(() => {})); + let waited = false; + const done = pass().then(() => { waited = true; }); + // eslint-disable-next-line no-await-in-loop + for (let i = 0; i < 1000 && !delayStub.called; i += 1) await Promise.resolve(); + sinon.assert.calledOnceWithExactly(delayStub, 3_600_000); + expect(waited, 'still waiting').to.equal(false); + + upnpService.removalState.emit(event); + await done; + + expect(waited).to.equal(true); + sinon.assert.calledOnce(setImmediateStub); + }); + } + it('removes what the router kept of a test mapping it refused', async () => { testMappingRefused(refused(718, false)); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 637c92bc23..c4175f4421 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -88,7 +88,7 @@ module.exports = { gatewayUrl: '', nodeIp: '', removalRetryFirstMs: 30 * 1000, - removalRetryMaxMs: 30 * 60 * 1000, + removalRetryMaxMs: 60 * 1000, }, benchmark: { host: '127.0.0.1', diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index b79080e79c..e11f61ec69 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -534,7 +534,7 @@ describe('upnpService tests', () => { expect(result).to.equal(true); expect([...upnpService.pendingRemovals.keys()]).to.deep.equal(['16130/TCP']); - expect(upnpService.testRemovalPending()).to.equal(true); + expect(upnpService.removalPending()).to.equal(true); upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); upnpService.pendingRemovals.clear(); }); @@ -1141,7 +1141,7 @@ describe('upnpService tests', () => { describe('pending removal tests', () => { const FIRST_MS = 30 * 1000; - const MAX_MS = 30 * 60 * 1000; + const MAX_MS = 60 * 1000; let clock; let createMappingStub; let removeMappingStub; @@ -1188,12 +1188,12 @@ describe('upnpService tests', () => { expect(removals().length, 'nothing before the first wait').to.equal(1); await clock.tickAsync(1); expect(removals().length, 'asked again after the first wait').to.equal(2); - await clock.tickAsync(2 * FIRST_MS - 1); + await clock.tickAsync(MAX_MS - 1); expect(removals().length, 'the wait doubles').to.equal(2); await clock.tickAsync(1); expect(removals().length).to.equal(3); - await clock.tickAsync(4 * FIRST_MS); - expect(removals().length, 'removed on the fourth ask').to.equal(4); + await clock.tickAsync(MAX_MS); + expect(removals().length, 'removed on the fourth ask, the wait capped').to.equal(4); expect(upnpService.pendingRemovals.size).to.equal(0); await clock.tickAsync(MAX_MS * 2); @@ -1205,15 +1205,15 @@ describe('upnpService tests', () => { removeMappingStub.rejects(refused()); await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); - // 30s, 1m, 2m, 4m, 8m, 16m, then 30m each - await clock.tickAsync((30 + 60 + 120 + 240 + 480 + 960) * 1000); - expect(removals().length).to.equal(7); + // 30s, then 1m each + await clock.tickAsync(FIRST_MS); + expect(removals().length).to.equal(2); await clock.tickAsync(MAX_MS - 1); - expect(removals().length).to.equal(7); + expect(removals().length).to.equal(2); await clock.tickAsync(1); - expect(removals().length).to.equal(8); + expect(removals().length).to.equal(3); await clock.tickAsync(MAX_MS * 10); - expect(removals().length, 'still asking').to.equal(18); + expect(removals().length, 'still asking').to.equal(13); expect(upnpService.pendingRemovals.has('123/TCP')).to.equal(true); }); @@ -1287,46 +1287,57 @@ describe('upnpService tests', () => { expect(upnpService.pendingRemovals.size).to.equal(0); }); - it('should ask now for a test mapping\'s pending removal, and say whether it has gone', async () => { - await mapTcpOnly(123, 'Flux_Test_App'); - removeMappingStub.onCall(0).rejects(refused()); - removeMappingStub.onCall(1).rejects(refused()); - await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); - - expect(await upnpService.askPendingTestRemovals(), 'refused again').to.equal(false); - expect(removals().length, 'asked at once, not after the wait').to.equal(2); - expect(upnpService.testRemovalPending()).to.equal(true); - - expect(await upnpService.askPendingTestRemovals(), 'removed').to.equal(true); - expect(removals().length).to.equal(3); - expect(upnpService.pendingRemovals.size).to.equal(0); - await clock.tickAsync(MAX_MS); - expect(removals().length, 'no scheduled ask left behind').to.equal(3); - }); - - it('should ask now for no app mapping\'s pending removal', async () => { - await mapTcpOnly(123, 'Flux_App_web'); - removeMappingStub.onCall(0).rejects(refused()); - await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); - - expect(await upnpService.askPendingTestRemovals()).to.equal(true); - expect(removals().length).to.equal(1); - expect(upnpService.pendingRemovals.has('123/TCP')).to.equal(true); - }); - - it('should hold back new test mappings while a test mapping\'s removal is pending, and only then', async () => { + it('should say a removal is pending until the router has removed it, whatever the mapping', async () => { await mapTcpOnly(123, 'Flux_App_web'); removeMappingStub.onCall(0).rejects(refused()); await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); - expect(upnpService.testRemovalPending(), 'an app mapping holds nothing back').to.equal(false); - - await mapTcpOnly(124, 'Flux_Test_App'); - removeMappingStub.onCall(1).rejects(refused()); - await upnpService.removeMapUpnpPort(124, 'Flux_Test_App'); - expect(upnpService.testRemovalPending()).to.equal(true); + expect(upnpService.removalPending()).to.equal(true); await clock.tickAsync(FIRST_MS); - expect(upnpService.testRemovalPending(), 'removed on the next ask').to.equal(false); + expect(upnpService.removalPending(), 'removed on the next ask').to.equal(false); + }); + + it('should say pending when the first removal is refused, and clear when the last goes', async () => { + const events = []; + const onPending = () => events.push('pending'); + const onClear = () => events.push('clear'); + upnpService.removalState.on('pending', onPending); + upnpService.removalState.on('clear', onClear); + try { + await mapTcpOnly(123, 'Flux_App_web'); + await mapTcpOnly(124, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + removeMappingStub.onCall(2).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + await upnpService.removeMapUpnpPort(124, 'Flux_Test_App'); + expect(events, 'once, for the first').to.deep.equal(['pending']); + + await clock.tickAsync(FIRST_MS); + expect(events, 'one still pending').to.deep.equal(['pending']); + await clock.tickAsync(MAX_MS); + expect(events).to.deep.equal(['pending', 'clear']); + } finally { + upnpService.removalState.off('pending', onPending); + upnpService.removalState.off('clear', onClear); + } + }); + + it('should clear when the port of the last pending removal is mapped again', async () => { + const onClear = sinon.spy(); + upnpService.removalState.on('clear', onClear); + try { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + sinon.assert.calledOnce(onClear); + expect(upnpService.removalPending()).to.equal(false); + } finally { + upnpService.removalState.off('clear', onClear); + } }); }); From a0bbd96d2efc5253576b51d23c9c1b093bf62e9a Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 08:10:01 +0100 Subject: [PATCH 137/369] test(harness): 3003 a node whose router refuses to remove its mappings is out of service until it does The UPnP stub records each request's protocol and description, and a suite can change how many removals a gateway refuses without resetting its table. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 20 ++- .../runner/framework/external-http-control.js | 13 +- ...appings-is-out-of-service-until-it-does.js | 166 ++++++++++++++++++ 3 files changed, 197 insertions(+), 2 deletions(-) create mode 100644 test-infra/runner/tests/3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 53186b9b56..1b8b01fa62 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -768,7 +768,13 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { const action = String(req.get('SOAPAction') || '').replace(/"/g, '').split('#')[1] || ''; const arg = (name) => (String(req.body).match(new RegExp(`<${name}>([^<]*)`)) || [])[1] ?? ''; const sameEntry = (m) => m.externalPort === Number(arg('NewExternalPort')) && m.protocol === arg('NewProtocol').toUpperCase(); - gateway.requests.push({ action, port: Number(arg('NewExternalPort')) || null, at: new Date().toISOString() }); + gateway.requests.push({ + action, + port: Number(arg('NewExternalPort')) || null, + protocol: arg('NewProtocol').toUpperCase() || null, + description: arg('NewPortMappingDescription') || null, + at: new Date().toISOString(), + }); switch (action) { case 'GetExternalIPAddress': soapReply(res, action, { NewExternalIPAddress: gateway.externalIp ?? clientAddress(req) }); @@ -1031,6 +1037,18 @@ control.post('/upnp-gateway', (req, res) => { res.json({ ok: true }); }); +// Set how many DeletePortMapping requests a node's gateway refuses from now on, +// keeping its table and the requests it has seen. +control.post('/upnp-gateway/:ip/refuse-deletes', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.refuseDeletes = Number(req.body?.count ?? 0); + res.json({ ok: true }); +}); + control.get('/upnp-gateway/:ip', (req, res) => { res.json(state.upnpGateways[req.params.ip] ?? null); }); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index 67305f0596..1dfe69a0c2 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -209,9 +209,20 @@ export async function enableUpnpGateway(ip, { if (!res.ok) throw new Error(`enableUpnpGateway: ${JSON.stringify(res)}`); } +/** + * Set how many DeletePortMapping requests a node's gateway refuses from now on, keeping its + * table and the requests it has seen. + * @param {string} ip + * @param {number} count - Infinity refuses every one + */ +export async function setUpnpRefuseDeletes(ip, count) { + const res = await post(`/upnp-gateway/${ip}/refuse-deletes`, { count: Number.isFinite(count) ? count : Number.MAX_SAFE_INTEGER }); + if (!res.ok) throw new Error(`setUpnpRefuseDeletes: ${JSON.stringify(res)}`); +} + /** * The gateway a node is behind: its port mappings and the actions the node sent it, each with - * the external port it named (null for an action that names none). + * the external port, protocol and description it named (null for one it names none of). * @param {string} ip * @returns {Promise<{externalIp: string|null, mappings: object[], requests: object[]}|null>} */ diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js new file mode 100644 index 0000000000..53ed294069 --- /dev/null +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js @@ -0,0 +1,166 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { pushBusybox } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { enableUpnpGateway, setUpnpRefuseDeletes, upnpGatewayState } from '../framework/external-http-control.js'; +import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; +import { + waitFor, waitForAppInstalled, waitForAppRemoved, waitForBootSettled, +} from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; +import { authenticate } from '../auth.js'; +import { fluxTeamKey } from '../framework/keys.js'; + +const subnet = getSubnetConfig(); + +// A node behind UPnP whose router refuses to remove its mappings is out of +// service until the router removes them. Its availability check waits ten +// minutes after a passing test and ten in DOS, so only UPnP reaching it can +// start a pass sooner: an app's removal the router refuses starts the next pass +// at once, each pass while it stays refused makes no test mapping and counts +// toward the DOS, and the router taking the removal starts the pass that tests +// and releases the node. + +const NODE = 0; +const LONG_WAIT_MS = 10 * 60 * 1000; +const APPS_PORT_CHECK = { + timeoutsMs: { + default: LONG_WAIT_MS, error: 5000, failure: 1000, dos: LONG_WAIT_MS, appError: 5000, + }, + thresholds: { dos: 100, portsHighEdge: 3, portsLowEdge: 1 }, +}; +// 25 counted passes, 5 s apart, reach the threshold of 100 +const DOS_WITHIN_MS = 4 * 60 * 1000; +// the harness's longest removal retry (8 s) and one pass +const RELEASED_WITHIN_MS = 90 * 1000; +const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; +const LISTEN_PORT = 8080; + +const dosOf = async (client) => (await client.getDOSState()).data ?? {}; + +describe('a node whose router refuses to remove its mappings is out of service until it does', function () { + let env; + dumpLogsOnFailure(() => env); + let node; + let nodeIp; + let appName; + let appPort; + const gateway = () => upnpGatewayState(nodeIp); + const testMappings = (requests, sinceMs = 0) => requests.filter((r) => r.action === 'AddPortMapping' + && r.description === 'Flux_Test_App' && Date.parse(r.at) >= sinceMs); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, + nodeConfigOverrides: { [NODE]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } } }, + }); + node = env.clients[NODE]; + nodeIp = subnet.nodeIp(NODE + 1); + // the first boot writes userconfig.js (its identity); a node with an api port + // in it verifies UPnP when it starts + await waitForBootSettled(node); + await enableUpnpGateway(nodeIp); + await setNodeApiPort(node.container, 16127); + await restartFluxos(node.container); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + + const { requests } = await gateway(); + expect(requests.some((r) => r.action === 'AddPortMapping' && r.description === 'Flux_UPNP_Mapping_Test'), 'canary: the node verified UPnP on its gateway').to.equal(true); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('tests its ports over TCP alone, and waits the long wait after a test that passed', async function () { + this.timeout(300000); + let state = { requests: [] }; + await waitFor(async () => { + state = await gateway(); + const [first] = testMappings(state.requests); + return Boolean(first) && state.requests.some((r) => r.action === 'DeletePortMapping' + && r.port === first.port && Date.parse(r.at) >= Date.parse(first.at)); + }, { timeout: 240000, interval: 2000, label: 'a test mapping made and removed' }); + + const made = testMappings(state.requests); + expect(made.map((r) => r.protocol), 'TCP only').to.deep.equal(made.map(() => 'TCP')); + + // a failed test is retried within seconds; a passed one only after the long wait + const quietFrom = Date.now(); + await new Promise((resolve) => { setTimeout(resolve, 30000); }); + expect(testMappings((await gateway()).requests, quietFrom), 'no test mapping in the long wait').to.have.length(0); + expect((await dosOf(node)).dosMessage ?? null).to.not.equal(UPNP_DOS); + }); + + it('is out of service, with no new test mapping, once its router has refused an app\'s removal for as many passes as the threshold takes', async function () { + this.timeout(DOS_WITHIN_MS + 400000); + await pushBusybox('e2eupnprefuse', 'v1', 'busybox', { + entrypoint: ['/bin/busybox', 'nc', '-lk', '-p', String(LISTEN_PORT), '-e', '/bin/busybox', 'echo', 'ok'], + }); + appName = `e2eupnprefuse${Date.now()}`; + const app = await buildSeedableApp({ + name: appName, + compose: [{ + name: appName, + description: 'holds a port on the router', + repotag: `${REGISTRY_REPO_HOST}/e2eupnprefuse:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [LISTEN_PORT], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + [appPort] = app.spec.compose[0].ports; + const mark = node.getLastEventId(); + await seedSpawnerApp(env, app, [NODE]); + await waitForAppInstalled(node, appName, 240000, { afterId: mark }); + expect((await gateway()).mappings.filter((m) => m.externalPort === appPort).map((m) => m.protocol).sort(), 'the app\'s port mapped').to.deep.equal(['TCP', 'UDP']); + + await setUpnpRefuseDeletes(nodeIp, Infinity); + const refusedFrom = Date.now(); + const auth = await authenticate(node.url, fluxTeamKey()); + const res = await fetch(`${node.url}/apps/appremove/${appName}`, { headers: { zelidauth: auth.zelidauth } }); + await res.text(); + await waitForAppRemoved(node, appName, 120000); + + await waitFor(async () => (await dosOf(node)).dosMessage === UPNP_DOS, { + timeout: DOS_WITHIN_MS, interval: 2000, label: 'the node out of service for its router', + }); + + const { requests, mappings } = await gateway(); + const refusedRemovals = requests.filter((r) => r.action === 'DeletePortMapping' && r.port === appPort && r.refused === 501); + expect(refusedRemovals.length, 'canary: the router refused the app\'s removal').to.be.at.least(2); + expect(mappings.some((m) => m.externalPort === appPort), 'the app\'s mapping still on the router').to.equal(true); + expect(testMappings(requests, refusedFrom), 'no test mapping while a removal is refused').to.have.length(0); + }); + + it('is back in service soon after its router removes the mapping', async function () { + this.timeout(RELEASED_WITHIN_MS + 60000); + const takenFrom = Date.now(); + await setUpnpRefuseDeletes(nodeIp, 0); + + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { + timeout: RELEASED_WITHIN_MS, interval: 2000, label: 'the node back in service', + }); + + const { requests, mappings } = await gateway(); + expect(mappings.some((m) => m.externalPort === appPort), 'the app\'s mapping removed').to.equal(false); + expect(testMappings(requests, takenFrom).length, 'a test ran once the router took the removal').to.be.at.least(1); + }); +}); From 8d8567a07a521016ac66fb076cde83bcec710688 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 08:29:38 +0100 Subject: [PATCH 138/369] fix(upnp): a mapping the router refuses wakes the availability check upnpService's routerEvents says 'mappingRefused' each time it counts a refusal as the router's own, beside 'removalPending' and 'removalsClear', and each starts the check's next pass at once: a router that starts refusing is looked at without waiting out a passing test's hour. A pass that counted a router refusal, saw one during the pass, or left a removal pending waits no longer than an error's wait (a DOS's while out of service), so the pass that finds the router working again comes soon. Every wait between passes is the one a wake ends, and a wake only ends it. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/availabilityChecker.js | 50 ++++++++++-------- ZelBack/src/services/upnpService.js | 18 ++++--- ...es-it-is-out-of-service-until-it-stops.js} | 0 tests/unit/availabilityChecker.test.js | 51 +++++++++++++++++-- tests/unit/upnpService.test.js | 31 ++++++++--- 5 files changed, 114 insertions(+), 36 deletions(-) rename test-infra/runner/tests/{3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js => 3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js} (100%) diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index fce3189c30..022e8372b0 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -100,8 +100,10 @@ function releaseOwnDos(dosState) { let wakeNextPass = null; /** - * Waits `ms` before the next pass, or less: a UPnP removal becoming pending, or - * the last one going, starts the next pass at once. + * Waits `ms` before the next pass, or less: the router refusing a mapping, a + * UPnP removal becoming pending, or the last one going, starts the next pass at + * once. Every wait between passes is this one, and a wake only ends it: there + * is never more than one pass. * @param {number} ms * @returns {Promise} */ @@ -118,8 +120,9 @@ function wakeCheck() { if (wakeNextPass) wakeNextPass(); } -upnpService.removalState.on('pending', wakeCheck); -upnpService.removalState.on('clear', wakeCheck); +upnpService.routerEvents.on('mappingRefused', wakeCheck); +upnpService.routerEvents.on('removalPending', wakeCheck); +upnpService.routerEvents.on('removalsClear', wakeCheck); /** * Whether the router has refused a mapping for a reason of its own - a full @@ -177,7 +180,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); if (!syncStatus.data.synced) { log.info('Flux Node daemon not synced. Application checks are disabled'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -187,7 +190,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (!isNodeConfirmed) { log.info('Flux Node not Confirmed. Application checks are disabled'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -195,7 +198,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const localSocketAddress = await fluxNetworkHelper.getLocalSocketAddress(); if (!localSocketAddress) { log.info('No Public IP found. Application checks are disabled'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -203,7 +206,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const installedAppsRes = await installedAppsFn(); if (installedAppsRes.status !== 'success') { log.error('Failed to get installed Apps'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -245,7 +248,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isPortBanned) { log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} is banned`); setNextPort(); - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -255,7 +258,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isPortUpnpBanned) { log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} is UPNP banned`); setNextPort(); - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -274,7 +277,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (appPorts.includes(dosState.testingPort)) { log.info(`checkMyAppsAvailability - Skipped checking ${dosState.testingPort} - in use`); setNextPort(); - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -284,13 +287,13 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // such node, which the retry below already handles. const remoteSocketAddress = await networkStateService.getRandomExternalObserver(localSocketAddress); if (!remoteSocketAddress) { - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } if (failedNodesTestPortsCache.has(remoteSocketAddress)) { - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -313,7 +316,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin skipHttpServer: true, }); const upnpDelay = dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error; - await serviceHelper.delay(upnpDelay); + await waitForNextPass(upnpDelay); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -341,7 +344,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin skipUpnp: !isUpnp, skipHttpServer: true, }); - await serviceHelper.delay(timeouts.error); + await waitForNextPass(timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -388,7 +391,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin }); if (!resMyAppAvailability) { - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -402,7 +405,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (!['success', 'error'].includes(responseStatus)) { log.warn(`checkMyAppsAvailability - Unexpected response status: ${responseStatus}`); - await serviceHelper.delay(timeouts.error); + await waitForNextPass(timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -459,14 +462,21 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin log.error(`checkMyAppsAvailability - Count: ${portsNotWorking.size}. portsNotWorking: ${JSON.stringify(Array.from(portsNotWorking))}`); } - // a removal refused during this pass gave no wake: the check was not waiting - if (isUpnp && upnpService.removalPending()) waitMs = Math.min(waitMs, timeouts.error); + // A router that refused this node a mapping or has yet to remove one is + // looked at again soon, so the pass that finds it working again comes + // soon; a refusal during this pass gave no wake, the check not waiting. + const routerFaulting = routerRefused + || upnpService.routerRefusals() > dosState.routerRefusalsSeen + || upnpService.removalPending(); + if (isUpnp && routerFaulting) { + waitMs = Math.min(waitMs, dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); + } await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp }); log.error(`checkMyAppsAvailability - Error: ${error}`); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } } diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 6933f6c70d..2b590049fa 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -55,9 +55,10 @@ const mappedProtocols = new Map(); const pendingRemovals = new Map(); let pendingRemovalTurn = Promise.resolve(); -// Emits 'pending' when a removal becomes pending with none pending before, and -// 'clear' when the last pending removal goes. -const removalState = new EventEmitter(); +// Emits 'removalPending' when a removal becomes pending with none pending +// before, 'removalsClear' when the last pending removal goes, and +// 'mappingRefused' each time a refusal is counted in routerRefusals. +const routerEvents = new EventEmitter(); /** * Makes a removal pending under `key`, in place of any pending there. @@ -68,7 +69,7 @@ function setPendingRemoval(key, pending) { const wasClear = pendingRemovals.size === 0; clearTimeout(pendingRemovals.get(key)?.timer); pendingRemovals.set(key, pending); - if (wasClear) removalState.emit('pending'); + if (wasClear) routerEvents.emit('removalPending'); } /** @@ -80,7 +81,7 @@ function deletePendingRemoval(key) { if (!pending) return; clearTimeout(pending.timer); pendingRemovals.delete(key); - if (pendingRemovals.size === 0) removalState.emit('clear'); + if (pendingRemovals.size === 0) routerEvents.emit('removalsClear'); } /** @@ -213,7 +214,10 @@ async function mapProtocol(port, protocol, description, { counted = true } = {}) const code = answered ? error.code : null; // no answer, with no code, is the router's too const routerWide = !PORT_REFUSALS.has(code) && !(heldElsewhere && code !== NO_PORT_MAPS_AVAILABLE); - if (routerWide && counted) routerRefusalCount += 1; + if (routerWide && counted) { + routerRefusalCount += 1; + routerEvents.emit('mappingRefused'); + } return { mapped: false, code, reason: `${protocol} ${port} not mapped: ${error.message}`, routerWide, }; @@ -917,7 +921,7 @@ module.exports = { mappedProtocols, pendingRemovals, removalPending, - removalState, + routerEvents, CLEANUP_SLOT_MS, BOOT_LISTING_STAGGER_MS, mapPortApi, diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js similarity index 100% rename from test-infra/runner/tests/3003-a-node-whose-router-refuses-to-remove-its-mappings-is-out-of-service-until-it-does.js rename to test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 8ca19e9e7a..2a0c6cc327 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -447,6 +447,38 @@ describe('availabilityChecker tests', () => { sinon.assert.calledWith(delayStub.lastCall, 60_000); }); + it('waits no longer than an error\'s wait after a passing test in a pass that counted a router refusal', async () => { + routerRefusesOthers(1); + + await pass(); + + sinon.assert.calledOnce(portTestStub); + sinon.assert.calledWith(delayStub.lastCall, 60_000); + }); + + it('waits no longer than an error\'s wait after a pass the router refused a mapping during', async () => { + portTestStub.callsFake(async () => { + routerRefusesOthers(1); + return { data: { status: 'success', data: { message: 'Ports reachable' } } }; + }); + + await pass(); + + expect(mockDosState.portFailureScore ?? 0, 'counted by the next pass').to.equal(0); + sinon.assert.calledWith(delayStub.lastCall, 60_000); + }); + + it('waits a DOS\'s wait after a pass that counted a router refusal while out of service', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + routerRefusesOthers(1); + + await pass(); + + sinon.assert.calledWith(delayStub.lastCall, 300_000); + }); + it('waits the whole wait after a passing test that left nothing pending', async () => { await pass(); @@ -454,8 +486,8 @@ describe('availabilityChecker tests', () => { }); // eslint-disable-next-line no-restricted-syntax - for (const event of ['pending', 'clear']) { - it(`starts its next pass at once when UPnP says its removals are ${event}`, async () => { + for (const event of ['mappingRefused', 'removalPending', 'removalsClear']) { + it(`starts its next pass at once on UPnP's ${event}`, async () => { delayStub.returns(new Promise(() => {})); let waited = false; const done = pass().then(() => { waited = true; }); @@ -464,7 +496,8 @@ describe('availabilityChecker tests', () => { sinon.assert.calledOnceWithExactly(delayStub, 3_600_000); expect(waited, 'still waiting').to.equal(false); - upnpService.removalState.emit(event); + upnpService.routerEvents.emit(event); + upnpService.routerEvents.emit(event); await done; expect(waited).to.equal(true); @@ -472,6 +505,18 @@ describe('availabilityChecker tests', () => { }); } + it('starts no pass on a wake while it is not waiting', async () => { + portTestStub.callsFake(async () => { + upnpService.routerEvents.emit('removalsClear'); + return { data: { status: 'success', data: { message: 'Ports reachable' } } }; + }); + + await pass(); + + sinon.assert.calledOnce(setImmediateStub); + sinon.assert.calledWith(delayStub.lastCall, 3_600_000); + }); + it('removes what the router kept of a test mapping it refused', async () => { testMappingRefused(refused(718, false)); diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index e11f61ec69..60d492f131 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -830,6 +830,25 @@ describe('upnpService tests', () => { expect([...upnpService.mappedProtocols.get(123)]).to.deep.equal(['TCP']); }); + it('should signal each refusal it counts as the router\'s, and no other', async () => { + const onRefused = sinon.spy(); + upnpService.routerEvents.on('mappingRefused', onRefused); + try { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); + getMappingStub.resolves(entry({ private: { host: '192.168.1.20', port: 123 }, local: false })); + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + sinon.assert.notCalled(onRefused); + + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.resolves(null); + await upnpService.mapUpnpPort(124, 'Flux_App_web'); + await upnpService.mapUpnpPort(125, 'Flux_App_web'); + sinon.assert.calledTwice(onRefused); + } finally { + upnpService.routerEvents.off('mappingRefused', onRefused); + } + }); + it('should not ask for UDP when TCP is refused', async () => { createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); @@ -1301,8 +1320,8 @@ describe('upnpService tests', () => { const events = []; const onPending = () => events.push('pending'); const onClear = () => events.push('clear'); - upnpService.removalState.on('pending', onPending); - upnpService.removalState.on('clear', onClear); + upnpService.routerEvents.on('removalPending', onPending); + upnpService.routerEvents.on('removalsClear', onClear); try { await mapTcpOnly(123, 'Flux_App_web'); await mapTcpOnly(124, 'Flux_Test_App'); @@ -1318,14 +1337,14 @@ describe('upnpService tests', () => { await clock.tickAsync(MAX_MS); expect(events).to.deep.equal(['pending', 'clear']); } finally { - upnpService.removalState.off('pending', onPending); - upnpService.removalState.off('clear', onClear); + upnpService.routerEvents.off('removalPending', onPending); + upnpService.routerEvents.off('removalsClear', onClear); } }); it('should clear when the port of the last pending removal is mapped again', async () => { const onClear = sinon.spy(); - upnpService.removalState.on('clear', onClear); + upnpService.routerEvents.on('removalsClear', onClear); try { await mapTcpOnly(123, 'Flux_App_web'); removeMappingStub.onCall(0).rejects(refused()); @@ -1336,7 +1355,7 @@ describe('upnpService tests', () => { sinon.assert.calledOnce(onClear); expect(upnpService.removalPending()).to.equal(false); } finally { - upnpService.removalState.off('clear', onClear); + upnpService.routerEvents.off('removalsClear', onClear); } }); }); From f669e53a11b2fae556ebb4b20929c1b27751e1fc Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 08:29:42 +0100 Subject: [PATCH 139/369] test(harness): 3003 covers a full router refusing an app's mapping The suite is now "a node whose router refuses it is out of service until it stops": its check waits ten minutes after a passing test and five in DOS, a full router's refusal of an app's mapping puts it out of service well inside the first, and it is back at its next pass once the router has room. The UPnP stub can change a gateway's capacity without resetting its table, and marks a mapping it refuses for being full. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 13 ++ .../runner/framework/external-http-control.js | 11 ++ ...ses-it-is-out-of-service-until-it-stops.js | 112 ++++++++++++------ 3 files changed, 103 insertions(+), 33 deletions(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 1b8b01fa62..60e29b448b 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -786,6 +786,7 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { return; } if (!held && gateway.capacity !== null && gateway.mappings.length >= gateway.capacity) { + gateway.requests[gateway.requests.length - 1].refused = 728; soapFault(res, 728, 'NoPortMapsAvailable'); return; } @@ -1049,6 +1050,18 @@ control.post('/upnp-gateway/:ip/refuse-deletes', (req, res) => { res.json({ ok: true }); }); +// Set how many mappings a node's gateway holds from now on (null: no limit), +// keeping its table and the requests it has seen. +control.post('/upnp-gateway/:ip/capacity', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.capacity = req.body?.capacity ?? null; + res.json({ ok: true }); +}); + control.get('/upnp-gateway/:ip', (req, res) => { res.json(state.upnpGateways[req.params.ip] ?? null); }); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index 1dfe69a0c2..5487a67333 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -220,6 +220,17 @@ export async function setUpnpRefuseDeletes(ip, count) { if (!res.ok) throw new Error(`setUpnpRefuseDeletes: ${JSON.stringify(res)}`); } +/** + * Set how many mappings a node's gateway holds from now on, keeping its table and the requests + * it has seen. A new mapping past it is refused with 728, carried as `refused: 728` in its state. + * @param {string} ip + * @param {number|null} capacity - null for no limit + */ +export async function setUpnpCapacity(ip, capacity) { + const res = await post(`/upnp-gateway/${ip}/capacity`, { capacity }); + if (!res.ok) throw new Error(`setUpnpCapacity: ${JSON.stringify(res)}`); +} + /** * The gateway a node is behind: its port mappings and the actions the node sent it, each with * the external port, protocol and description it named (null for one it names none of). diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js index 53ed294069..dacda116ec 100644 --- a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js @@ -8,7 +8,9 @@ import { restartFluxos, setNodeApiPort } from '../framework/container.js'; import { pushBusybox } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; -import { enableUpnpGateway, setUpnpRefuseDeletes, upnpGatewayState } from '../framework/external-http-control.js'; +import { + enableUpnpGateway, setUpnpCapacity, setUpnpRefuseDeletes, upnpGatewayState, +} from '../framework/external-http-control.js'; import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; import { waitFor, waitForAppInstalled, waitForAppRemoved, waitForBootSettled, @@ -19,19 +21,24 @@ import { fluxTeamKey } from '../framework/keys.js'; const subnet = getSubnetConfig(); -// A node behind UPnP whose router refuses to remove its mappings is out of -// service until the router removes them. Its availability check waits ten -// minutes after a passing test and ten in DOS, so only UPnP reaching it can -// start a pass sooner: an app's removal the router refuses starts the next pass -// at once, each pass while it stays refused makes no test mapping and counts -// toward the DOS, and the router taking the removal starts the pass that tests -// and releases the node. +// A node behind UPnP whose router refuses it - to remove a mapping, or to make +// one - is out of service until the router stops. Its availability check waits +// ten minutes after a passing test and five in DOS, so only UPnP reaching it +// can start a pass sooner than that. +// +// An app's removal the router refuses starts the next pass at once; each pass +// while it stays refused makes no test mapping and counts toward the DOS, and +// the router taking the removal starts the pass that tests and releases the +// node. An app's mapping a full router refuses starts the next pass at once; +// that pass's own test mapping is refused too, and so is every one after it +// until the router has room, when the next pass releases the node. const NODE = 0; -const LONG_WAIT_MS = 10 * 60 * 1000; +const PASSED_WAIT_MS = 10 * 60 * 1000; +const DOS_WAIT_MS = 5 * 60 * 1000; const APPS_PORT_CHECK = { timeoutsMs: { - default: LONG_WAIT_MS, error: 5000, failure: 1000, dos: LONG_WAIT_MS, appError: 5000, + default: PASSED_WAIT_MS, error: 5000, failure: 1000, dos: DOS_WAIT_MS, appError: 5000, }, thresholds: { dos: 100, portsHighEdge: 3, portsLowEdge: 1 }, }; @@ -39,12 +46,15 @@ const APPS_PORT_CHECK = { const DOS_WITHIN_MS = 4 * 60 * 1000; // the harness's longest removal retry (8 s) and one pass const RELEASED_WITHIN_MS = 90 * 1000; +if (DOS_WITHIN_MS >= PASSED_WAIT_MS || RELEASED_WITHIN_MS >= DOS_WAIT_MS) { + throw new Error('3003: each budget must end before the wait only a wake can cut short'); +} const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; const LISTEN_PORT = 8080; const dosOf = async (client) => (await client.getDOSState()).data ?? {}; -describe('a node whose router refuses to remove its mappings is out of service until it does', function () { +describe('a node whose router refuses it is out of service until it stops', function () { let env; dumpLogsOnFailure(() => env); let node; @@ -54,6 +64,24 @@ describe('a node whose router refuses to remove its mappings is out of service u const gateway = () => upnpGatewayState(nodeIp); const testMappings = (requests, sinceMs = 0) => requests.filter((r) => r.action === 'AddPortMapping' && r.description === 'Flux_Test_App' && Date.parse(r.at) >= sinceMs); + const buildApp = (name) => buildSeedableApp({ + name, + compose: [{ + name, + description: 'holds a port on the router', + repotag: `${REGISTRY_REPO_HOST}/e2eupnprefuse:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [LISTEN_PORT], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); before(async function () { this.timeout(600000); @@ -72,6 +100,9 @@ describe('a node whose router refuses to remove its mappings is out of service u await setNodeApiPort(node.container, 16127); await restartFluxos(node.container); await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await pushBusybox('e2eupnprefuse', 'v1', 'busybox', { + entrypoint: ['/bin/busybox', 'nc', '-lk', '-p', String(LISTEN_PORT), '-e', '/bin/busybox', 'echo', 'ok'], + }); const { requests } = await gateway(); expect(requests.some((r) => r.action === 'AddPortMapping' && r.description === 'Flux_UPNP_Mapping_Test'), 'canary: the node verified UPnP on its gateway').to.equal(true); @@ -104,28 +135,8 @@ describe('a node whose router refuses to remove its mappings is out of service u it('is out of service, with no new test mapping, once its router has refused an app\'s removal for as many passes as the threshold takes', async function () { this.timeout(DOS_WITHIN_MS + 400000); - await pushBusybox('e2eupnprefuse', 'v1', 'busybox', { - entrypoint: ['/bin/busybox', 'nc', '-lk', '-p', String(LISTEN_PORT), '-e', '/bin/busybox', 'echo', 'ok'], - }); appName = `e2eupnprefuse${Date.now()}`; - const app = await buildSeedableApp({ - name: appName, - compose: [{ - name: appName, - description: 'holds a port on the router', - repotag: `${REGISTRY_REPO_HOST}/e2eupnprefuse:v1`, - ports: [], - domains: [''], - environmentParameters: [], - commands: [], - containerPorts: [LISTEN_PORT], - containerData: '/tmp', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', - }], - }); + const app = await buildApp(appName); [appPort] = app.spec.compose[0].ports; const mark = node.getLastEventId(); await seedSpawnerApp(env, app, [NODE]); @@ -150,7 +161,7 @@ describe('a node whose router refuses to remove its mappings is out of service u expect(testMappings(requests, refusedFrom), 'no test mapping while a removal is refused').to.have.length(0); }); - it('is back in service soon after its router removes the mapping', async function () { + it('is back in service soon after its router removes the mapping, sooner than its wait in DOS', async function () { this.timeout(RELEASED_WITHIN_MS + 60000); const takenFrom = Date.now(); await setUpnpRefuseDeletes(nodeIp, 0); @@ -163,4 +174,39 @@ describe('a node whose router refuses to remove its mappings is out of service u expect(mappings.some((m) => m.externalPort === appPort), 'the app\'s mapping removed').to.equal(false); expect(testMappings(requests, takenFrom).length, 'a test ran once the router took the removal').to.be.at.least(1); }); + + it('is out of service once a full router refuses an app\'s mapping, sooner than its wait after a passing test', async function () { + this.timeout(DOS_WITHIN_MS + 400000); + // the pass that released the node passed its test: the check is in its wait after one + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { timeout: 30000, interval: 2000 }); + const { mappings } = await gateway(); + await setUpnpCapacity(nodeIp, mappings.length); + const fullFrom = Date.now(); + const name = `e2eupnpfull${Date.now()}`; + const app = await buildApp(name); + const [port] = app.spec.compose[0].ports; + await seedSpawnerApp(env, app, [NODE]); + + await waitFor(async () => (await dosOf(node)).dosMessage === UPNP_DOS, { + timeout: DOS_WITHIN_MS, interval: 2000, label: 'the node out of service for its full router', + }); + + const { requests } = await gateway(); + const refused = requests.filter((r) => r.action === 'AddPortMapping' && r.refused === 728 && Date.parse(r.at) >= fullFrom); + expect(refused.some((r) => r.port === port), 'canary: the router refused the app\'s mapping').to.equal(true); + expect(refused.some((r) => r.description === 'Flux_Test_App'), 'the check\'s own test mapping refused too').to.equal(true); + }); + + it('is back in service at its next pass once the router has room', async function () { + this.timeout(DOS_WAIT_MS + 120000); + const roomFrom = Date.now(); + await setUpnpCapacity(nodeIp, null); + + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { + timeout: DOS_WAIT_MS + 60000, interval: 2000, label: 'the node back in service', + }); + + const made = testMappings((await gateway()).requests, roomFrom); + expect(made.some((r) => r.refused === undefined), 'a test mapping the router took').to.equal(true); + }); }); From 5c1c62f343eb0a5d1f3ba57f7a8ae4adf3b1dbe2 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 09:30:40 +0100 Subject: [PATCH 140/369] chore(upnp): nat-upnp ~2.0.5 Co-Authored-By: Claude Opus 5.5 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 6dd6c00528..dee54d682d 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,7 @@ "@aws-sdk/client-ecr": "^3.922.0", "@azure/identity": "^4.13.0", "@isaacs/ttlcache": "~1.4.1", - "@runonflux/nat-upnp": "~2.0.4", + "@runonflux/nat-upnp": "~2.0.5", "apicache": "~1.6.3", "archiver": "~7.0.1", "axios": "~1.13.6", From d0f4261d086a430ebf3751a71c096f17b1e385cc Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 10:17:52 +0100 Subject: [PATCH 141/369] fix(upnp): the cleanup gives the router's requests whole-millisecond timeouts AbortSignal.timeout throws on a fraction, so a budget that is not a whole number of milliseconds, or one that halves to a fraction, is rounded down before the listing's and each removal's timeout are taken from it. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 6 ++++-- tests/unit/upnpService.test.js | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 9e57dd494f..c70ffdfc3f 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -396,9 +396,11 @@ function msUntilCleanupSlot(apiport, now = Date.now()) { * @returns {Promise} how many mappings were removed */ async function removeStaleMappings(heldPorts, budgetMs) { - const deadline = monotonicMs() + budgetMs; + // AbortSignal.timeout takes whole milliseconds only + const wholeBudgetMs = Math.floor(budgetMs); + const deadline = monotonicMs() + wholeBudgetMs; const left = () => deadline - monotonicMs(); - const listingMs = budgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, budgetMs / 2); + const listingMs = wholeBudgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, Math.floor(wholeBudgetMs / 2)); const mappings = await client.getMappings({ local: true, signal: AbortSignal.timeout(listingMs) }); const keep = new Set([...await heldPorts()].map(Number)); const stale = mappings.filter((mapping) => Number.isInteger(mapping.public.port) diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index a65332f130..cece85d419 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -692,6 +692,17 @@ describe('upnpService tests', () => { sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP', signal: sinon.match.instanceOf(AbortSignal) }); }); + it('should give the router\'s requests whole-millisecond timeouts from a budget that halves to a fraction', async () => { + routerHas([mapping(31000, 'Flux_App_gone')]); + + const removed = await upnpService.removeStaleMappings(async () => [16127], 29_999); + + expect(removed).to.equal(1); + const timeouts = AbortSignal.timeout.getCalls().map((call) => call.args[0]); + expect(timeouts).to.have.length(2); + timeouts.forEach((ms) => expect(Number.isInteger(ms), `${ms} ms`).to.equal(true)); + }); + it('should remove nothing from an empty table', async () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); From 1412f13948bdfe8121aac57d79b77ac4ee3a99c5 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 10:17:53 +0100 Subject: [PATCH 142/369] fix(upnp): the cleanup gives the router's requests whole-millisecond timeouts AbortSignal.timeout throws on a fraction, so a budget that is not a whole number of milliseconds, or one that halves to a fraction, is rounded down before the listing's and each removal's timeout are taken from it. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 6 ++++-- tests/unit/upnpService.test.js | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 2b590049fa..892a81a42a 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -660,9 +660,11 @@ function msUntilCleanupSlot(apiport, now = Date.now()) { * @returns {Promise} how many mappings were removed */ async function removeOwnMappings(select, budgetMs, what) { - const deadline = monotonicMs() + budgetMs; + // AbortSignal.timeout takes whole milliseconds only + const wholeBudgetMs = Math.floor(budgetMs); + const deadline = monotonicMs() + wholeBudgetMs; const left = () => deadline - monotonicMs(); - const listingMs = budgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, budgetMs / 2); + const listingMs = wholeBudgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, Math.floor(wholeBudgetMs / 2)); const mappings = await client.getMappings({ local: true, signal: AbortSignal.timeout(listingMs) }); const chosen = (await select(mappings)).filter((mapping) => Number.isInteger(mapping.public.port)); let removed = 0; diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 60d492f131..e88d9ec6dc 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -1454,6 +1454,17 @@ describe('upnpService tests', () => { expect(removedPorts).to.deep.equal([31000, 31001, 31002]); }); + it('should give the router\'s requests whole-millisecond timeouts from a budget that halves to a fraction', async () => { + routerHas([mapping(31000, 'Flux_App_gone')]); + + const removed = await upnpService.removeStaleMappings(async () => [16127], 29_999); + + expect(removed).to.equal(1); + const timeouts = AbortSignal.timeout.getCalls().map((call) => call.args[0]); + expect(timeouts).to.have.length(2); + timeouts.forEach((ms) => expect(Number.isInteger(ms), `${ms} ms`).to.equal(true)); + }); + it('should remove nothing from an empty table', async () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); From f834c8491eee5926795a6a1673af872c24e01557 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 10:42:31 +0100 Subject: [PATCH 143/369] test(registry): a test waits for the rebuild to start however long its height read takes The rebuild reads the scanned height from mongo before it reaches the stubbed rebuild, and the tests waited for it a fixed 1000 setImmediate turns. A read slower than those few milliseconds left the rebuild unstarted and the next test waiting behind the one still in flight. The wait now ends when the stub has been called, bounded only by the test's own timeout. Co-Authored-By: Claude Opus 5.5 --- tests/unit/registryManager.test.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index b704df4b2e..d7aec36c89 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1203,11 +1203,12 @@ describe('registryManager tests', () => { promise.then(() => { state.done = true; }, () => { state.done = true; }); return state; }; - // the rebuild reads the scanned height from mongo before it reaches the stub + // the rebuild reads the scanned height from mongo before it reaches the + // stub, so this waits for the stub however long that read takes const rebuildsStarted = async (count) => { - for (let i = 0; i < 1000 && rebuildStub.callCount < count; i += 1) { + while (rebuildStub.callCount < count) { // eslint-disable-next-line no-await-in-loop - await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setTimeout(resolve, 1); }); } expect(rebuildStub.callCount).to.equal(count); }; From 92404063c5812ffcbac29cd9bf76a1b8fabb622d Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 10:53:19 +0100 Subject: [PATCH 144/369] fix(upnp): a pending removal looks the entry up before each retry Before asking the router again to remove a mapping, upnpService asks whether the router still maps it to this node. One it no longer holds, or holds for something else, ends the removal with nothing sent; so does a lookup the router answers with a fault, since a router finds this node's own entries (no lease) whenever it holds them. Only a lookup the router does not answer leaves the removal pending, to be asked again. A router that answers a removal of an entry it no longer holds with anything but 714 can no longer keep the removal pending, and with it the node out of service. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 31 +++++++++- tests/unit/upnpService.test.js | 87 ++++++++++++++++++++++++++++- 2 files changed, 114 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 892a81a42a..1da6da3b57 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -492,17 +492,44 @@ function dropPendingRemovals(port) { ['TCP', 'UDP'].forEach((protocol) => deletePendingRemoval(`${port}/${protocol}`)); } +/** + * Whether the router still maps one protocol of a port to this node. A router + * finds this node's own entries, which carry no lease, whenever it holds them, + * so a lookup it answers with a fault is taken as holding none. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @returns {Promise} null when the router did not answer + */ +async function routerMapsHere(port, protocol) { + try { + return mapsHere(await client.getMapping({ public: port, protocol }), port); + } catch (error) { + if (error instanceof natUpnp.UpnpError) return false; + log.error(`UPnP - ${protocol} ${port} not looked up: ${error.message}`); + return null; + } +} + /** * Asks the router once more to remove a pending removal's mapping, in turn * behind any other pending removal, and schedules the next ask if it refuses - * again. + * again. The router is asked first whether the mapping is still this node's: + * one it no longer holds, or holds for something else, ends the removal with + * nothing sent, and a router that does not answer is asked again later. * @param {object} pending */ function askPendingRemoval(pending) { pendingRemovalTurn = pendingRemovalTurn.then(async () => { const key = `${pending.port}/${pending.protocol}`; if (pendingRemovals.get(key) !== pending) return; - if (await removeProtocol(pending.port, pending.protocol)) { + const held = await routerMapsHere(pending.port, pending.protocol); + if (pendingRemovals.get(key) !== pending) return; + if (held === false) { + deletePendingRemoval(key); + log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) no longer this node's on the router, nothing to remove`); + return; + } + if (held && await removeProtocol(pending.port, pending.protocol)) { deletePendingRemoval(key); log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); return; diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index e88d9ec6dc..3937dfd5dc 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -1163,14 +1163,30 @@ describe('upnpService tests', () => { const MAX_MS = 60 * 1000; let clock; let createMappingStub; + let getMappingStub; let removeMappingStub; const refused = () => new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping'); + // the router's entry for a port, to this node unless overridden + const entry = (port, protocol, overrides = {}) => ({ + public: { host: '', port }, + private: { host: '192.168.1.10', port }, + protocol: protocol.toLowerCase(), + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + ...overrides, + }); + const lookups = () => getMappingStub.getCalls().map((call) => `${call.args[0].public}/${call.args[0].protocol}`); const removals = () => removeMappingStub.getCalls().map((call) => `${call.args[0].public}/${call.args[0].protocol}`); - // a test port mapped TCP only: its UDP refused + // a test port mapped TCP only: its UDP refused, held by another machine const mapTcpOnly = async (port, description) => { createMappingStub.withArgs(sinon.match({ public: port, protocol: 'UDP' })).rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + getMappingStub.withArgs(sinon.match({ public: port, protocol: 'UDP' })) + .resolves(entry(port, 'UDP', { private: { host: '192.168.1.11', port }, local: false })); await upnpService.mapUpnpPort(port, description); + getMappingStub.resetHistory(); }; beforeEach(() => { @@ -1182,7 +1198,8 @@ describe('upnpService tests', () => { sinon.stub(log, 'error'); sinon.stub(log, 'info'); createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); - sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping') + .callsFake(async ({ public: port, protocol }) => entry(port, protocol)); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); clock = sinon.useFakeTimers(); }); @@ -1358,6 +1375,72 @@ describe('upnpService tests', () => { upnpService.routerEvents.off('removalsClear', onClear); } }); + + it('should look the entry up before each ask, and remove it only while it is this node\'s', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + expect(lookups(), 'the first removal asks nothing first').to.deep.equal([]); + + await clock.tickAsync(FIRST_MS); + + expect(lookups()).to.deep.equal(['123/TCP']); + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should end a removal with nothing sent when the router no longer holds the entry, whatever it answers to a removal', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.resolves(null); + + await clock.tickAsync(FIRST_MS); + + expect(removals(), 'no removal after the lookup').to.deep.equal(['123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + await clock.tickAsync(MAX_MS * 5); + expect(lookups(), 'asked nothing more').to.have.length(1); + }); + + it('should end a removal with nothing sent when the router holds the port for something else', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.callsFake(async ({ public: port, protocol }) => entry(port, protocol, { private: { host: '192.168.1.11', port }, local: false })); + + await clock.tickAsync(FIRST_MS); + + expect(removals()).to.deep.equal(['123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + }); + + it('should end a removal with nothing sent when the router answers the lookup with a fault', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'GetSpecificPortMappingEntry')); + + await clock.tickAsync(FIRST_MS); + + expect(removals()).to.deep.equal(['123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + }); + + it('should keep a removal pending, with nothing sent, while the router does not answer the lookup', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.onCall(0).rejects(new Error('timeout of 10000ms exceeded')); + + await clock.tickAsync(FIRST_MS); + expect(removals(), 'nothing sent on no answer').to.deep.equal(['123/TCP']); + expect(upnpService.removalPending()).to.equal(true); + + await clock.tickAsync(MAX_MS); + expect(removals(), 'removed once the router answers').to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + }); }); describe('msUntilCleanupSlot tests', () => { From 7fd9a3a8d4b53b641bf30e90d8481570455f71d7 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 10:53:57 +0100 Subject: [PATCH 145/369] test(harness): 3003 a router that carries out a removal but answers it with a fault leaves the node in service The UPnP stub can answer every removal with a fault while still carrying it out, and a removal of an entry it no longer holds with the same fault rather than 714. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 20 +++++++++ .../runner/framework/external-http-control.js | 12 ++++++ ...ses-it-is-out-of-service-until-it-stops.js | 43 ++++++++++++++++++- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 60e29b448b..b5c7b69304 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -812,6 +812,13 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { } const before = gateway.mappings.length; gateway.mappings = gateway.mappings.filter((m) => !sameEntry(m)); + // a router that removes the entry and answers with a fault, and answers a + // removal of one it does not hold with the same fault, never 714 + if (gateway.deleteFault) { + gateway.requests[gateway.requests.length - 1].refused = gateway.deleteFault; + soapFault(res, gateway.deleteFault, 'ActionFailed'); + return; + } if (gateway.mappings.length === before) soapFault(res, 714, 'NoSuchEntryInArray'); else soapReply(res, action, {}); return; @@ -1050,6 +1057,19 @@ control.post('/upnp-gateway/:ip/refuse-deletes', (req, res) => { res.json({ ok: true }); }); +// Set the fault a node's gateway answers every removal with from now on, while +// still carrying it out (null: answer as the spec says), keeping its table and +// the requests it has seen. +control.post('/upnp-gateway/:ip/delete-fault', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.deleteFault = req.body?.code ?? null; + res.json({ ok: true }); +}); + // Set how many mappings a node's gateway holds from now on (null: no limit), // keeping its table and the requests it has seen. control.post('/upnp-gateway/:ip/capacity', (req, res) => { diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index 5487a67333..ac85db7a32 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -220,6 +220,18 @@ export async function setUpnpRefuseDeletes(ip, count) { if (!res.ok) throw new Error(`setUpnpRefuseDeletes: ${JSON.stringify(res)}`); } +/** + * Set the fault a node's gateway answers every removal with from now on, while still carrying + * it out, and the same fault for a removal of an entry it does not hold, where the spec says + * 714. Keeps its table and the requests it has seen; each such answer carries `refused: `. + * @param {string} ip + * @param {number|null} code - null answers as the spec says + */ +export async function setUpnpDeleteFault(ip, code) { + const res = await post(`/upnp-gateway/${ip}/delete-fault`, { code }); + if (!res.ok) throw new Error(`setUpnpDeleteFault: ${JSON.stringify(res)}`); +} + /** * Set how many mappings a node's gateway holds from now on, keeping its table and the requests * it has seen. A new mapping past it is refused with 728, carried as `refused: 728` in its state. diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js index dacda116ec..ad9190af83 100644 --- a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js @@ -9,7 +9,7 @@ import { pushBusybox } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { - enableUpnpGateway, setUpnpCapacity, setUpnpRefuseDeletes, upnpGatewayState, + enableUpnpGateway, setUpnpCapacity, setUpnpDeleteFault, setUpnpRefuseDeletes, upnpGatewayState, } from '../framework/external-http-control.js'; import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; import { @@ -31,7 +31,10 @@ const subnet = getSubnetConfig(); // the router taking the removal starts the pass that tests and releases the // node. An app's mapping a full router refuses starts the next pass at once; // that pass's own test mapping is refused too, and so is every one after it -// until the router has room, when the next pass releases the node. +// until the router has room, when the next pass releases the node. A router +// that carries out a removal but answers it with a fault - and answers the +// same for an entry it no longer holds - leaves nothing pending: the retry +// finds the entry gone, and the node stays in service. const NODE = 0; const PASSED_WAIT_MS = 10 * 60 * 1000; @@ -209,4 +212,40 @@ describe('a node whose router refuses it is out of service until it stops', func const made = testMappings((await gateway()).requests, roomFrom); expect(made.some((r) => r.refused === undefined), 'a test mapping the router took').to.equal(true); }); + + it('stays in service when its router carries out an app\'s removal but answers it with a fault', async function () { + this.timeout(DOS_WITHIN_MS + 400000); + const name = `e2eupnpfault${Date.now()}`; + const app = await buildApp(name); + const [port] = app.spec.compose[0].ports; + const mark = node.getLastEventId(); + await seedSpawnerApp(env, app, [NODE]); + await waitForAppInstalled(node, name, 240000, { afterId: mark }); + expect((await gateway()).mappings.some((m) => m.externalPort === port), 'the app\'s port mapped').to.equal(true); + + await setUpnpDeleteFault(nodeIp, 501); + const faultFrom = Date.now(); + const auth = await authenticate(node.url, fluxTeamKey()); + const res = await fetch(`${node.url}/apps/appremove/${name}`, { headers: { zelidauth: auth.zelidauth } }); + await res.text(); + await waitForAppRemoved(node, name, 120000); + + // a pending removal would count every 5 s pass and reach the DOS inside this window + const watchedUntil = Date.now() + DOS_WITHIN_MS; + while (Date.now() < watchedUntil) { + // eslint-disable-next-line no-await-in-loop + expect((await dosOf(node)).dosMessage ?? null, 'never out of service').to.not.equal(UPNP_DOS); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 5000); }); + } + await setUpnpDeleteFault(nodeIp, null); + + const { requests, mappings } = await gateway(); + const ofPort = requests.filter((r) => r.port === port && Date.parse(r.at) >= faultFrom); + expect(ofPort.some((r) => r.action === 'DeletePortMapping' && r.refused === 501), 'canary: the router answered the removal with a fault').to.equal(true); + expect(mappings.some((m) => m.externalPort === port), 'canary: and carried it out').to.equal(false); + const lookup = ofPort.findIndex((r) => r.action === 'GetSpecificPortMappingEntry'); + expect(lookup, 'the retry looked the entry up').to.be.at.least(0); + expect(ofPort.slice(lookup).filter((r) => r.action === 'DeletePortMapping'), 'no removal sent once it was found gone').to.have.length(0); + }); }); From ecef333c234de828c6e5ec99bd6803e08f437b42 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 11:37:54 +0100 Subject: [PATCH 146/369] fix(upnp): a mapping counts as this node's until the router shows it is not A pending removal ends with nothing sent only when the lookup shows the router holds no entry for the port, or holds it for something else. A lookup the router answers with a fault, or does not answer, settles nothing: the mapping is still counted as this node's and the removal is sent, so a mapping is never left on the router on a lookup that failed. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/upnpService.js | 13 +++++------ tests/unit/upnpService.test.js | 34 ++++++++++++++++++++++------- 2 files changed, 32 insertions(+), 15 deletions(-) diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 1da6da3b57..865f94984f 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -493,18 +493,16 @@ function dropPendingRemovals(port) { } /** - * Whether the router still maps one protocol of a port to this node. A router - * finds this node's own entries, which carry no lease, whenever it holds them, - * so a lookup it answers with a fault is taken as holding none. + * Whether the router still maps one protocol of a port to this node. * @param {number} port * @param {string} protocol 'TCP' or 'UDP' - * @returns {Promise} null when the router did not answer + * @returns {Promise} null when the router did not say: a lookup + * it answered with a fault, or did not answer */ async function routerMapsHere(port, protocol) { try { return mapsHere(await client.getMapping({ public: port, protocol }), port); } catch (error) { - if (error instanceof natUpnp.UpnpError) return false; log.error(`UPnP - ${protocol} ${port} not looked up: ${error.message}`); return null; } @@ -515,7 +513,8 @@ async function routerMapsHere(port, protocol) { * behind any other pending removal, and schedules the next ask if it refuses * again. The router is asked first whether the mapping is still this node's: * one it no longer holds, or holds for something else, ends the removal with - * nothing sent, and a router that does not answer is asked again later. + * nothing sent. A lookup it does not settle leaves the mapping counted as + * this node's, and the removal is sent. * @param {object} pending */ function askPendingRemoval(pending) { @@ -529,7 +528,7 @@ function askPendingRemoval(pending) { log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) no longer this node's on the router, nothing to remove`); return; } - if (held && await removeProtocol(pending.port, pending.protocol)) { + if (await removeProtocol(pending.port, pending.protocol)) { deletePendingRemoval(key); log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); return; diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index 3937dfd5dc..b74d394869 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -1415,31 +1415,49 @@ describe('upnpService tests', () => { expect(upnpService.removalPending()).to.equal(false); }); - it('should end a removal with nothing sent when the router answers the lookup with a fault', async () => { + it('should send the removal when the router answers the lookup with a fault, and keep it pending while the removal is refused', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'GetSpecificPortMappingEntry')); + + await clock.tickAsync(FIRST_MS); + + expect(removals(), 'a lookup it did not settle leaves the mapping counted as this node\'s').to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.removalPending()).to.equal(true); + + removeMappingStub.resolves(); + await clock.tickAsync(MAX_MS); + expect(removals()).to.have.length(3); + expect(upnpService.removalPending(), 'ended once the router removed it').to.equal(false); + }); + + it('should end a removal whose lookup the router answers with a fault once it answers the removal with no such entry', async () => { await mapTcpOnly(123, 'Flux_App_web'); removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(new natUpnp.UpnpError(714, 'NoSuchEntryInArray', 'DeletePortMapping')); await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); getMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'GetSpecificPortMappingEntry')); await clock.tickAsync(FIRST_MS); - expect(removals()).to.deep.equal(['123/TCP']); + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); expect(upnpService.removalPending()).to.equal(false); }); - it('should keep a removal pending, with nothing sent, while the router does not answer the lookup', async () => { + it('should send the removal while the router does not answer the lookup, and keep it pending while the removal is refused', async () => { await mapTcpOnly(123, 'Flux_App_web'); - removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.rejects(refused()); await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); - getMappingStub.onCall(0).rejects(new Error('timeout of 10000ms exceeded')); + getMappingStub.rejects(new Error('timeout of 10000ms exceeded')); await clock.tickAsync(FIRST_MS); - expect(removals(), 'nothing sent on no answer').to.deep.equal(['123/TCP']); + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); expect(upnpService.removalPending()).to.equal(true); + removeMappingStub.resolves(); await clock.tickAsync(MAX_MS); - expect(removals(), 'removed once the router answers').to.deep.equal(['123/TCP', '123/TCP']); - expect(upnpService.removalPending()).to.equal(false); + expect(upnpService.removalPending(), 'ended once the router removed it').to.equal(false); }); }); From f68b2fdc6d504362e12d01a19acedd076fcfa335 Mon Sep 17 00:00:00 2001 From: David White Date: Tue, 20 Oct 2026 11:38:10 +0100 Subject: [PATCH 147/369] test(harness): 3003 a lookup the router answers with a fault leaves the mapping counted as the node's The UPnP stub can answer every GetSpecificPortMappingEntry with a fault, whether or not it holds the entry. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 18 ++++++++ .../runner/framework/external-http-control.js | 12 +++++ ...ses-it-is-out-of-service-until-it-stops.js | 45 ++++++++++++++++++- 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index b5c7b69304..3610c45830 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -824,6 +824,11 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { return; } case 'GetSpecificPortMappingEntry': { + if (gateway.lookupFault) { + gateway.requests[gateway.requests.length - 1].refused = gateway.lookupFault; + soapFault(res, gateway.lookupFault, 'ActionFailed'); + return; + } const entry = gateway.mappings.find(sameEntry); if (!entry) { soapFault(res, 714, 'NoSuchEntryInArray'); @@ -1057,6 +1062,19 @@ control.post('/upnp-gateway/:ip/refuse-deletes', (req, res) => { res.json({ ok: true }); }); +// Set the fault a node's gateway answers every lookup with from now on, whether +// or not it holds the entry (null: answer as the spec says), keeping its table +// and the requests it has seen. +control.post('/upnp-gateway/:ip/lookup-fault', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.lookupFault = req.body?.code ?? null; + res.json({ ok: true }); +}); + // Set the fault a node's gateway answers every removal with from now on, while // still carrying it out (null: answer as the spec says), keeping its table and // the requests it has seen. diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index ac85db7a32..2487fcafe6 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -220,6 +220,18 @@ export async function setUpnpRefuseDeletes(ip, count) { if (!res.ok) throw new Error(`setUpnpRefuseDeletes: ${JSON.stringify(res)}`); } +/** + * Set the fault a node's gateway answers every GetSpecificPortMappingEntry with from now on, + * whether or not it holds the entry. Keeps its table and the requests it has seen; each such + * answer carries `refused: `. + * @param {string} ip + * @param {number|null} code - null answers as the spec says + */ +export async function setUpnpLookupFault(ip, code) { + const res = await post(`/upnp-gateway/${ip}/lookup-fault`, { code }); + if (!res.ok) throw new Error(`setUpnpLookupFault: ${JSON.stringify(res)}`); +} + /** * Set the fault a node's gateway answers every removal with from now on, while still carrying * it out, and the same fault for a removal of an entry it does not hold, where the spec says diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js index ad9190af83..6318c43dbc 100644 --- a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js @@ -9,7 +9,8 @@ import { pushBusybox } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { - enableUpnpGateway, setUpnpCapacity, setUpnpDeleteFault, setUpnpRefuseDeletes, upnpGatewayState, + enableUpnpGateway, setUpnpCapacity, setUpnpDeleteFault, setUpnpLookupFault, setUpnpRefuseDeletes, + upnpGatewayState, } from '../framework/external-http-control.js'; import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; import { @@ -34,7 +35,10 @@ const subnet = getSubnetConfig(); // until the router has room, when the next pass releases the node. A router // that carries out a removal but answers it with a fault - and answers the // same for an entry it no longer holds - leaves nothing pending: the retry -// finds the entry gone, and the node stays in service. +// finds the entry gone, and the node stays in service. A lookup the router +// answers with a fault settles nothing: the mapping still counts as the +// node's, the node is out of service while the router refuses to remove it, +// and back once the router has. const NODE = 0; const PASSED_WAIT_MS = 10 * 60 * 1000; @@ -248,4 +252,41 @@ describe('a node whose router refuses it is out of service until it stops', func expect(lookup, 'the retry looked the entry up').to.be.at.least(0); expect(ofPort.slice(lookup).filter((r) => r.action === 'DeletePortMapping'), 'no removal sent once it was found gone').to.have.length(0); }); + + it('counts a mapping as its own while the router answers its lookup with a fault, until the router removes it', async function () { + this.timeout(DOS_WITHIN_MS + RELEASED_WITHIN_MS + 400000); + const name = `e2eupnplookup${Date.now()}`; + const app = await buildApp(name); + const [port] = app.spec.compose[0].ports; + const mark = node.getLastEventId(); + await seedSpawnerApp(env, app, [NODE]); + await waitForAppInstalled(node, name, 240000, { afterId: mark }); + expect((await gateway()).mappings.some((m) => m.externalPort === port), 'the app\'s port mapped').to.equal(true); + + await setUpnpLookupFault(nodeIp, 501); + await setUpnpRefuseDeletes(nodeIp, Infinity); + const faultFrom = Date.now(); + const auth = await authenticate(node.url, fluxTeamKey()); + const res = await fetch(`${node.url}/apps/appremove/${name}`, { headers: { zelidauth: auth.zelidauth } }); + await res.text(); + await waitForAppRemoved(node, name, 120000); + + await waitFor(async () => (await dosOf(node)).dosMessage === UPNP_DOS, { + timeout: DOS_WITHIN_MS, interval: 2000, label: 'the node out of service while the router keeps the mapping', + }); + let state = await gateway(); + const ofPort = () => state.requests.filter((r) => r.port === port && Date.parse(r.at) >= faultFrom); + expect(ofPort().some((r) => r.action === 'GetSpecificPortMappingEntry' && r.refused === 501), 'canary: a lookup answered with a fault').to.equal(true); + const firstLookup = ofPort().findIndex((r) => r.action === 'GetSpecificPortMappingEntry'); + expect(ofPort().slice(firstLookup).some((r) => r.action === 'DeletePortMapping'), 'the removal sent after a lookup that settled nothing').to.equal(true); + expect(state.mappings.some((m) => m.externalPort === port), 'the mapping still on the router').to.equal(true); + + await setUpnpRefuseDeletes(nodeIp, 0); + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { + timeout: RELEASED_WITHIN_MS, interval: 2000, label: 'the node back in service once the router removed the mapping', + }); + await setUpnpLookupFault(nodeIp, null); + state = await gateway(); + expect(state.mappings.some((m) => m.externalPort === port), 'the mapping removed').to.equal(false); + }); }); From e48c465ef81e99d2c29e097bbde7ef5bcda6ed59 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 08:22:11 +0100 Subject: [PATCH 148/369] feat(ports): a peer probes UDP and can answer for every port /flux/checkappavailability takes two opt-in fields. `udp: true` probes each port over UDP once TCP has passed and returns what replied in `udpAnswered`: the reply, null for silence, the port left out when no probe could be sent. `everyPort: true`, with `echo`, reads every port in parallel and returns an answer for each, null for a port that did not answer, instead of ending at the first failure. A caller sending neither gets the check it always got. FluxUdpTestServer is the UDP half of a port test: it answers a probe with the run's token, only to the peer being asked, never with more bytes than the probe, and at most a fixed number of times. The test port firewall rule takes its protocol. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxNetworkHelper.js | 170 ++++++++++++++++-- .../src/services/utils/fluxUdpTestServer.js | 116 ++++++++++++ tests/unit/fluxNetworkHelper.test.js | 15 ++ tests/unit/fluxUdpTestServer.test.js | 110 ++++++++++++ 4 files changed, 393 insertions(+), 18 deletions(-) create mode 100644 ZelBack/src/services/utils/fluxUdpTestServer.js create mode 100644 tests/unit/fluxUdpTestServer.test.js diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 19635393c1..31f4bcb74a 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -28,6 +28,7 @@ const fluxEventBus = require('./utils/fluxEventBus'); const { NON_PUBLIC_IPV4 } = require('./utils/nonPublicNetworks'); const ufw = require('./utils/ufw'); const ufwHelper = require('./utils/ufwHelper'); +const { UDP_PROBE_MAGIC, UDP_PROBE_BYTES } = require('./utils/fluxUdpTestServer'); const { normalizeSocketAddress, extractIp, extractPort, socketAddressesMatch, parseSocketAddress, ipsMatch, } = require('./utils/socketAddressUtils'); @@ -422,6 +423,98 @@ async function isPortOpen(ip, port, options = {}) { return connected; } +/** + * When each datagram of a UDP probe is sent, in ms from the first. UDP drops + * packets and loss comes in bursts, so the sends are spread rather than sent + * together. + */ +const UDP_PROBE_SCHEDULE_MS = [0, 250, 750, 1750]; + +/** How long a UDP probe waits for a reply after its first datagram. */ +const UDP_PROBE_DEADLINE_MS = 3_000; + +/** + * What a UDP port answered, capped, for the requester to judge - the UDP twin + * of portAnswered. + * + * UDP has no handshake, so silence is all a closed port and a lost datagram + * ever say. The requester's test server answers a probe with its secret, and + * only that tells the requester the port is reachable; this hands back what + * came, and the requester compares. + * + * The reply is taken from whatever source port it arrives from: a NAT in front + * of the requester may rewrite it. + * + * @param {string} ip - the requester's address + * @param {number} port - the port to probe + * @param {{schedule?: number[], deadline?: number}} [options] + * @returns {Promise} what it answered; null when probes + * went out and nothing came back; undefined when no probe could be sent, which + * says nothing about the port + */ +async function portAnsweredUdp(ip, port, options = {}) { + const schedule = options.schedule ?? UDP_PROBE_SCHEDULE_MS; + const deadline = options.deadline ?? UDP_PROBE_DEADLINE_MS; + + const probe = Buffer.alloc(UDP_PROBE_BYTES); + probe.write(UDP_PROBE_MAGIC, 'latin1'); + + const socket = dgram.createSocket('udp4'); + + return new Promise((resolve) => { + const timers = []; + let settled = false; + let sent = 0; + let failed = 0; + + const done = (answer) => { + if (settled) return; + settled = true; + timers.forEach(clearTimeout); + socket.close(); + resolve(answer); + }; + + const afterSend = (error) => { + if (!error) { + sent += 1; + return; + } + failed += 1; + if (failed === schedule.length) done(undefined); + }; + + socket.on('message', (message, rinfo) => { + if (rinfo.address.replace(/^::ffff:/i, '') !== ip) return; + done(message.subarray(0, MAX_ECHO_BYTES).toString('utf8')); + }); + socket.on('error', () => done(undefined)); + + schedule.forEach((at) => { + timers.push(setTimeout(() => socket.send(probe, port, ip, afterSend), at)); + }); + timers.push(setTimeout(() => done(sent ? null : undefined), deadline)); + }); +} + +/** + * Runs `task` once per item, at most `limit` at a time. + * @param {Array} items + * @param {number} limit + * @param {function(*): Promise} task + * @returns {Promise} + */ +async function forEachLimited(items, limit, task) { + const queue = [...items]; + async function worker() { + while (queue.length) { + // eslint-disable-next-line no-await-in-loop + await task(queue.shift()); + } + } + await Promise.all(Array.from({ length: Math.min(limit, queue.length) }, worker)); +} + /** * To perform a basic check of current FluxOS version. * @param {string} ip IP address. @@ -557,6 +650,14 @@ const MAX_TESTABLE_PORTS = 50; // how long one request can keep this node poking, and nothing tighter: the // honest list really can be that long. const NODE_SERVICE_PORTS_KEPT_ALIVE = 4; + +/** + * How many of one request's ports a peer probes at once, where it probes them + * in parallel: UDP always, TCP when asked for every port's answer. A port that + * never answers costs a full probe timeout, and 50 of them one after another + * would outlast the requester's wait. + */ +const PORT_PROBE_CONCURRENCY = 16; const MAX_KEEPALIVE_PORTS = config.fluxapps.maxAppsPerNode * MAX_TESTABLE_PORTS + NODE_SERVICE_PORTS_KEPT_ALIVE; /** @@ -651,15 +752,31 @@ async function checkAppAvailability(req, res) { // A requester that wants proof asks for it. One that does not - an older // node - gets exactly the check it always got. const echo = processedBody.echo === true; + // Every port read and answered for, a port that did not answer reading + // null, rather than the test ending at the first port that failed. Only + // with echo: the answers are what it returns. + const everyPort = echo && processedBody.everyPort === true; + // Each port probed over UDP as well, once TCP has passed. + const udp = processedBody.udp === true; const answered = {}; - // eslint-disable-next-line no-restricted-syntax - for (const port of ports) { + const testable = ports.filter((port) => { const iBP = isPortBanned(+port); const portNum = +port; const withinRange = portNum >= minPort && portNum <= maxPort; + if (!withinRange || iBP) { + log.error(`Flux App port ${port} is outside allowed range. minPort: ${minPort}, maxPort: ${maxPort}, isBanned: ${iBP}`); + } + return withinRange && !iBP; + }); - if (withinRange && !iBP) { + if (everyPort) { + await forEachLimited(testable, PORT_PROBE_CONCURRENCY, async (port) => { + answered[port] = await portAnswered(ip, port); + }); + } else { + // eslint-disable-next-line no-restricted-syntax + for (const port of testable) { if (echo) { // Read rather than merely reached. The requester compares. // eslint-disable-next-line no-await-in-loop @@ -675,10 +792,18 @@ async function checkAppAvailability(req, res) { throw new Error(`Flux Applications on ${ip}:${ipPort} are not available. Failed port: ${port}`); } } - } else { - log.error(`Flux App port ${port} is outside allowed range. minPort: ${minPort}, maxPort: ${maxPort}, isBanned: ${iBP}`); } } + + // A port this node could not send to is left out: that says nothing about + // the port, and the requester reads an absent port as no proof. + const udpAnswered = {}; + if (udp) { + await forEachLimited(testable, PORT_PROBE_CONCURRENCY, async (port) => { + const reply = await portAnsweredUdp(ip, +port); + if (reply !== undefined) udpAnswered[port] = reply; + }); + } const successResponse = messageHelper.createSuccessMessage(`Flux Applications on ${ip}:${ipPort} are available.`); // `answered` present at all is how the requester knows this peer READ the // ports rather than merely reaching them. Absent means no proof is @@ -686,6 +811,8 @@ async function checkAppAvailability(req, res) { // Added as a field rather than through createSuccessMessage, whose second // and third parameters are name and code. if (echo) successResponse.data.answered = answered; + // Present at all says this peer probed UDP, as `answered` does for TCP. + if (udp) successResponse.data.udpAnswered = udpAnswered; res.json(successResponse); } catch (error) { const errorResponse = messageHelper.createErrorMessage( @@ -2844,36 +2971,42 @@ async function allowPort(port) { return cmdStat; } -// The ufw rule that admits IPv4 TCP clients to a port FluxOS opens for a moment, -// to test it from outside, and nothing else. ufw counts a rule that names no -// protocol as the same rule as the matching half of an app port's +// The ufw rule that admits IPv4 clients of one protocol to a port FluxOS opens +// for a moment, to test it from outside, and nothing else. ufw counts a rule that +// names no protocol as the same rule as the matching half of an app port's // `allow `, so a test that opened and deleted one would delete that half; -// a TCP rule is a rule of its own. -const testPortRule = (port) => ['proto', 'tcp', 'from', '0.0.0.0/0', 'to', 'any', 'port', String(port)]; +// a rule naming its protocol is a rule of its own. +const testPortRule = (port, protocol) => ['proto', protocol, 'from', '0.0.0.0/0', 'to', 'any', 'port', String(port)]; + +const TEST_PORT_PROTOCOLS = ['tcp', 'udp']; /** - * Opens a port to IPv4 TCP clients while FluxOS tests it from outside: the only rule - * for the port that deleteTestPortRule deletes. + * Opens a port to IPv4 clients of one protocol while FluxOS tests it from + * outside: the only rule for the port that deleteTestPortRule deletes. * @param {number|string} port Port. + * @param {string} [protocol] 'tcp' or 'udp'. * @returns {Promise<{status: boolean, message: (string|null)}>} */ -async function allowTestPort(port) { +async function allowTestPort(port, protocol = 'tcp') { if (Number.isNaN(+port)) return { status: false, message: 'Port needs to be a number' }; - const ran = await ufw.runUfw(['allow', ...testPortRule(port)]); + if (!TEST_PORT_PROTOCOLS.includes(protocol)) return { status: false, message: 'Protocol needs to be tcp or udp' }; + const ran = await ufw.runUfw(['allow', ...testPortRule(port, protocol)]); if (ran.locked) return { status: false, message: 'ufw is locked by another ufw command' }; const cmdres = ran.stdout + ran.stderr; return { status: /added|updated|existing/.test(cmdres), message: cmdres }; } /** - * Deletes the rule allowTestPort writes, and no other rule for the port. A rule - * already gone counts as deleted. + * Deletes the rule allowTestPort writes for a protocol, and no other rule for + * the port. A rule already gone counts as deleted. * @param {number|string} port Port. + * @param {string} [protocol] 'tcp' or 'udp'. * @returns {Promise<{status: boolean, message: (string|null)}>} */ -async function deleteTestPortRule(port) { +async function deleteTestPortRule(port, protocol = 'tcp') { if (Number.isNaN(+port)) return { status: false, message: 'Port needs to be a number' }; - const ran = await ufw.runUfw(['delete', 'allow', ...testPortRule(port)]); + if (!TEST_PORT_PROTOCOLS.includes(protocol)) return { status: false, message: 'Protocol needs to be tcp or udp' }; + const ran = await ufw.runUfw(['delete', 'allow', ...testPortRule(port, protocol)]); if (ran.locked) return { status: false, message: 'ufw is locked by another ufw command' }; const cmdres = ran.stdout + ran.stderr; return { status: cmdres.includes('delete'), message: cmdres }; @@ -3543,6 +3676,7 @@ module.exports = { isPortOpen, portAnswered, MAX_ECHO_BYTES, + portAnsweredUdp, checkAppAvailability, verifySignedFluxnodeMessage, isPortEnterprise, diff --git a/ZelBack/src/services/utils/fluxUdpTestServer.js b/ZelBack/src/services/utils/fluxUdpTestServer.js new file mode 100644 index 0000000000..ef177f14d5 --- /dev/null +++ b/ZelBack/src/services/utils/fluxUdpTestServer.js @@ -0,0 +1,116 @@ +const dgram = require('node:dgram'); +const { once } = require('node:events'); + +/** + * Every probe a peer sends to a UDP port under test starts with this, and every + * reply from a test server starts with it too. + */ +const UDP_PROBE_MAGIC = 'FLUXPT'; + +/** + * The size of every probe. A reply is never larger: the test server answers a + * stranger who spoofs a peer's address, so it must not amplify what it is sent. + */ +const UDP_PROBE_BYTES = 64; + +/** + * The most replies one test server sends while it listens. A peer sends a + * handful of datagrams per port per attempt; this bounds what a flood of + * probes forged from the asked peer's address can make the server send. + */ +const MAX_REPLIES = 32; + +function noop() {} + +/** + * The reply the test server sends for a token. + * @param {string} token + * @returns {Buffer} + */ +function udpReply(token) { + return Buffer.from(`${UDP_PROBE_MAGIC} ${token}`); +} + +/** + * The UDP half of a port test: bound on the port under test while the test + * runs, and answering a peer's probe with the run's secret token. + * + * It answers only the peer this node is asking at the moment, set with + * `allow`. A UDP reply goes wherever the probe claims it came from, so + * answering anyone would make every port under test a reflector. + */ +class FluxUdpTestServer { + #socket = dgram.createSocket('udp4'); + + #reply = null; + + #allowed = null; + + #replies = 0; + + /** + * @param {string} token The secret this run's test servers answer with. + */ + constructor(token) { + this.#reply = udpReply(token); + if (this.#reply.length > UDP_PROBE_BYTES) { + throw new Error(`A UDP test token of ${token.length} characters does not fit a ${UDP_PROBE_BYTES} byte probe`); + } + this.#socket.on('message', (message, rinfo) => this.#answer(message, rinfo)); + // A bind error rejects listen(); a send that fails costs that one reply. + // Neither may reach the process as an unhandled 'error'. + this.#socket.on('error', noop); + } + + /** + * The peer whose probes this server answers from now on; null answers nobody. + * @param {string|null} ip + */ + allow(ip) { + this.#allowed = ip ? ip.replace(/^::ffff:/i, '') : null; + } + + /** + * Binds the port. Rejects with the bind error, EADDRINUSE among them. + * @param {number} port + * @param {string} [address] + * @returns {Promise} + */ + async listen(port, address) { + const bound = once(this.#socket, 'listening'); + this.#socket.bind(port, address); + await bound; + } + + /** @returns {{address: string, port: number}} */ + address() { + return this.#socket.address(); + } + + /** Unbinds the port. Safe to call on a server that never bound or already closed. */ + close() { + try { + this.#socket.close(); + } catch { + // ERR_SOCKET_DGRAM_NOT_RUNNING: closed already + } + } + + #answer(message, rinfo) { + if (this.#replies >= MAX_REPLIES) return; + if (!this.#allowed || rinfo.address.replace(/^::ffff:/i, '') !== this.#allowed) return; + if (message.length < UDP_PROBE_BYTES) return; + if (message.subarray(0, UDP_PROBE_MAGIC.length).toString('latin1') !== UDP_PROBE_MAGIC) return; + + this.#replies += 1; + this.#socket.send(this.#reply, rinfo.port, rinfo.address, noop); + } +} + +module.exports = { + FluxUdpTestServer, + UDP_PROBE_MAGIC, + UDP_PROBE_BYTES, + MAX_REPLIES, + udpReply, +}; diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index ee85aa3c6e..a6fb18eaae 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -2969,6 +2969,21 @@ describe('fluxNetworkHelper tests', () => { }); }); + it('opens and deletes a UDP test rule that names udp', async () => { + await fluxNetworkHelper.allowTestPort(31350, 'udp'); + await fluxNetworkHelper.deleteTestPortRule(31350, 'udp'); + expect(runUfw.args).to.deep.equal([ + [['allow', 'proto', 'udp', 'from', '0.0.0.0/0', 'to', 'any', 'port', '31350']], + [['delete', 'allow', 'proto', 'udp', 'from', '0.0.0.0/0', 'to', 'any', 'port', '31350']], + ]); + }); + + it('refuses a protocol other than tcp or udp, running nothing', async () => { + expect((await fluxNetworkHelper.allowTestPort(31350, 'any')).status).to.equal(false); + expect((await fluxNetworkHelper.deleteTestPortRule(31350, 'any')).status).to.equal(false); + sinon.assert.notCalled(runUfw); + }); + it('reports a held lock and refuses a port that is not a number', async () => { runUfw.resolves({ error: null, stdout: '', stderr: '', locked: true }); expect(await fluxNetworkHelper.allowTestPort(31350)).to.deep.equal({ status: false, message: 'ufw is locked by another ufw command' }); diff --git a/tests/unit/fluxUdpTestServer.test.js b/tests/unit/fluxUdpTestServer.test.js new file mode 100644 index 0000000000..e498043c36 --- /dev/null +++ b/tests/unit/fluxUdpTestServer.test.js @@ -0,0 +1,110 @@ +const dgram = require('node:dgram'); +const { once } = require('node:events'); +const { expect } = require('chai'); +const { + FluxUdpTestServer, UDP_PROBE_MAGIC, UDP_PROBE_BYTES, MAX_REPLIES, udpReply, +} = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); + +// The UDP half of the port test. A peer probes the port and hands back what +// replied; the requester looks for its secret in it. A UDP reply goes wherever +// the probe claims it came from, so what this server answers, and to whom, is +// the whole of its contract. +describe('FluxUdpTestServer', () => { + const TOKEN = 'c0ffee00'.repeat(4); + let server; + let port; + let client; + + const probe = (size = UDP_PROBE_BYTES, magic = UDP_PROBE_MAGIC) => { + const datagram = Buffer.alloc(size); + datagram.write(magic, 'latin1'); + return datagram; + }; + + // Sends a datagram from the client and collects what comes back in a short + // window: silence is an answer here, so the wait is the assertion's. + const exchange = async (datagram, count = 1) => { + const replies = []; + const onMessage = (message) => replies.push(message.toString()); + client.on('message', onMessage); + for (let i = 0; i < count; i += 1) { + client.send(datagram, port, '127.0.0.1'); + } + await new Promise((resolve) => { setTimeout(resolve, 150); }); + client.off('message', onMessage); + return replies; + }; + + beforeEach(async () => { + server = new FluxUdpTestServer(TOKEN); + await server.listen(0, '127.0.0.1'); + ({ port } = server.address()); + client = dgram.createSocket('udp4'); + client.bind(0, '127.0.0.1'); + await once(client, 'listening'); + }); + + afterEach(() => { + server.close(); + client.close(); + }); + + it('answers the allowed peer\'s probe with the token', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe())).to.deep.equal([udpReply(TOKEN).toString()]); + }); + + it('answers nobody until a peer is allowed', async () => { + expect(await exchange(probe())).to.deep.equal([]); + }); + + it('answers nobody but the allowed peer', async () => { + server.allow('198.51.100.7'); + + expect(await exchange(probe())).to.deep.equal([]); + }); + + it('reads an IPv4-mapped allowed address as the address it is', async () => { + server.allow('::ffff:127.0.0.1'); + + expect(await exchange(probe())).to.have.length(1); + }); + + it('never sends a reply larger than the probe it answers', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe(UDP_PROBE_BYTES - 1))).to.deep.equal([]); + expect(udpReply(TOKEN).length).to.be.at.most(UDP_PROBE_BYTES); + }); + + it('answers only a probe, not any datagram', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe(UDP_PROBE_BYTES, 'HELLO!'))).to.deep.equal([]); + }); + + it('stops answering after its reply cap', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe(), MAX_REPLIES + 8)).to.have.length(MAX_REPLIES); + }); + + it('refuses a token too long for a reply no larger than a probe', () => { + expect(() => new FluxUdpTestServer('x'.repeat(UDP_PROBE_BYTES))).to.throw(); + }); + + it('rejects listen when the port is taken', async () => { + const second = new FluxUdpTestServer(TOKEN); + let error = null; + await second.listen(port, '127.0.0.1').catch((e) => { error = e; }); + second.close(); + + expect(error && error.code).to.equal('EADDRINUSE'); + }); + + it('closes twice without throwing', () => { + server.close(); + expect(() => server.close()).to.not.throw(); + }); +}); From 0e337555cf6205307cb9cff67b41298ef79c9e36 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 08:22:12 +0100 Subject: [PATCH 149/369] feat(ports): the pre-install port test proves UDP Every app port is published for TCP and UDP, so the pre-install test now proves both. Each port gets a UDP test rule and a UDP test server beside the HTTP one, and the peer is asked to probe UDP. Our token heard over UDP proves a port, from any peer. A peer hearing nothing on a port is evidence against it until another peer proves it, and two distinct peers with standing evidence refuse the install as udpUnreachable. A peer that does not probe UDP gives no proof and is never a witness, so an install proven over TCP goes ahead as udpUnproven when no two peers agree. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appNetwork/portManager.js | 166 ++++++++++++-- tests/unit/portManager.test.js | 206 +++++++++++++++++- tests/unit/portTestEcho.test.js | 151 +++++++++++++ 3 files changed, 505 insertions(+), 18 deletions(-) diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 235573cde7..b6f2a4a4d4 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -11,6 +11,7 @@ const upnpService = require('../upnpService'); const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const fluxHttpTestServer = require('../utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../utils/fluxUdpTestServer'); const { localAppsInformation, globalAppsInformation } = require('../utils/appConstants'); const appUtilities = require('../utils/appUtilities'); const { Privilege, authOf } = require('../utils/privileges'); @@ -808,6 +809,66 @@ function portNotOurs(portsToTest, answered, token) { return at === -1 ? null : portsToTest[at]; } +/** + * Reads one peer's UDP answers: adds the ports our token came back from to + * those proven so far, and names the ports this peer heard nothing of ours on. + * + * Our token is proof, whichever peer heard it. Silence, or an answer without + * our token, is this peer's evidence against a port, and it stands only while + * no peer has proven that port: UDP loses datagrams, so one peer's silence on a + * port another peer heard our token from is loss. A port missing from the + * answers is one the peer could not send to, which says nothing. + * + * @param {number[]} portsToTest - the ports asked about + * @param {object} udpAnswered - port -> what that port replied, or null, from the peer + * @param {string} token - the secret this node's test servers answer with + * @param {Set} proven - ports proven so far; added to + * @returns {number[]} the ports this peer heard nothing of ours on + */ +function udpPortsNotOurs(portsToTest, udpAnswered, token, proven) { + const closed = []; + portsToTest.forEach((port) => { + const probed = port >= config.fluxapps.portMin && port <= config.fluxapps.portMax; + if (!probed) return; + + const reply = udpAnswered[port] !== undefined ? udpAnswered[port] : udpAnswered[String(port)]; + if (reply === undefined) return; + + if (typeof reply === 'string' && reply.includes(token)) { + proven.add(port); + } else { + closed.push(port); + } + }); + return closed; +} + +/** + * Each peer whose UDP evidence still stands, with the first port it heard + * nothing on that no peer has proven. + * @param {Map} udpDisagreements - peer -> ports it heard nothing on + * @param {Set} proven + * @returns {Map} + */ +function udpDisputes(udpDisagreements, proven) { + const disputes = new Map(); + udpDisagreements.forEach((ports, peer) => { + const port = ports.find((closed) => !proven.has(closed)); + if (port !== undefined) disputes.set(peer, port); + }); + return disputes; +} + +/** + * Whether every port a peer probes over UDP is proven. + * @param {number[]} portsToTest + * @param {Set} proven + * @returns {boolean} + */ +function everyUdpPortProven(portsToTest, proven) { + return portsToTest.every((port) => port < config.fluxapps.portMin || port > config.fluxapps.portMax || proven.has(port)); +} + /** * To check if app ports are available publicly before installation. * @@ -831,6 +892,7 @@ function portNotOurs(portsToTest, answered, token) { async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { const description = `Flux_App_${appName}`; const beforeAppInstallTestingServers = []; + const udpTestServers = []; // One secret for this whole test run, published by our own test servers and // never sent to the peer. The peer returns what it read; we compare. const portTestToken = crypto.randomBytes(16).toString('hex'); @@ -877,7 +939,9 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // now open this port properly and launch listening on it if (firewallActive) { // eslint-disable-next-line no-await-in-loop - await fluxNetworkHelper.allowTestPort(portToTest); + await fluxNetworkHelper.allowTestPort(portToTest, 'tcp'); + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.allowTestPort(portToTest, 'udp'); } if (isUPNP) { // eslint-disable-next-line no-await-in-loop @@ -914,6 +978,11 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { const error = await listening.catch((err) => err); if (error) throw error; + + const udpTestServer = new fluxUdpTestServer.FluxUdpTestServer(portTestToken); + udpTestServers.push(udpTestServer); + // eslint-disable-next-line no-await-in-loop + await udpTestServer.listen(portToTest); } await serviceHelper.delay(config.fluxapps.portTestPropagationDelayMs); @@ -931,6 +1000,8 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // it should find is deliberately not here: a peer that never learns it // cannot produce it without actually reaching us. echo: true, + // And to probe each port over UDP, handing back what replied. + udp: true, }; const stringData = JSON.stringify(data); // eslint-disable-next-line no-await-in-loop @@ -951,6 +1022,16 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // Peer address -> the port that peer said is not serving this node. Keyed by // peer so that redrawing the same one does not read as a second opinion. const disagreements = new Map(); + // Peer address -> the ports that peer heard nothing of ours on over UDP. + // Kept apart from TCP: a peer that reached every port over TCP and heard + // nothing over UDP is saying something else. + const udpDisagreements = new Map(); + // Our token came back over TCP from every port, from one peer. Proof, so no + // later peer's reading can outweigh it. + let tcpProven = false; + // Ports our token came back from over UDP, from any peer. + const udpProven = new Set(); + let sawNoUdpPeer = false; // Every peer already asked, so a redraw is ANOTHER peer rather than another // draw. Without this the picker can hand back the one just asked - which on // a budget of two attempts it often does - and a check counting distinct @@ -985,6 +1066,7 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { const askingIP = extractIp(randomSocketAddress); const askingIpPort = extractPort(randomSocketAddress); asked.push(randomSocketAddress); + udpTestServers.forEach((udpTestServer) => udpTestServer.allow(askingIP)); // first check against our IP address // eslint-disable-next-line no-await-in-loop @@ -996,6 +1078,7 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // What this attempt learned about one of our ports, or null when it // learned nothing at all. Only a port recorded here counts as a witness. let reportedPort = null; + let udpClosedPorts = []; if (!resMyAppAvailability) { // The peer never answered us. That is a fact about the peer, not about @@ -1029,9 +1112,9 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // token coming back settles a port on one peer's say-so, because only // this node could have produced it. log.warn(`checkInstallingAppPortAvailable - ${askingIP} could not reach port ${failedPort}`); - reportedPort = failedPort; + if (!tcpProven) reportedPort = failedPort; } else if (resMyAppAvailability.data.status === 'success') { - const { answered } = resMyAppAvailability.data.data || {}; + const { answered, udpAnswered } = resMyAppAvailability.data.data || {}; if (!answered) { // This peer is on older code: it reached the ports but did not read @@ -1046,13 +1129,27 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { if (notOurs === null) { // Our own token came back. Proof, not report - only this node could - // have produced it - so one peer settles it. + // have produced it - so one peer settles TCP. + tcpProven = true; + } else if (!tcpProven) { + log.info(`checkInstallingAppPortAvailable - ${askingIP} read something other than this node on port ${notOurs}`); + reportedPort = notOurs; + } + + if (udpAnswered && typeof udpAnswered === 'object') { + udpClosedPorts = udpPortsNotOurs(portsToTest, udpAnswered, portTestToken, udpProven); + if (udpClosedPorts.length) { + log.warn(`checkInstallingAppPortAvailable - ${askingIP} heard nothing of this node over UDP on port(s) ${udpClosedPorts.join(', ')}`); + } + } else { + // A peer on older code: it does not probe UDP. + sawNoUdpPeer = true; + } + + if (tcpProven && everyUdpPortProven(portsToTest, udpProven)) { decision = { ok: true, reason: 'proven', port: null }; break; } - - log.info(`checkInstallingAppPortAvailable - ${askingIP} read something other than this node on port ${notOurs}`); - reportedPort = notOurs; } else { // An answer in a shape this node does not understand is not an answer. sawSilentPeer = true; @@ -1060,6 +1157,31 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { continue; } + if (udpClosedPorts.length) { + udpDisagreements.set(askingIP, udpClosedPorts); + const disputed = udpDisputes(udpDisagreements, udpProven); + const udpRefused = refusedPort(disputed, PORT_TEST_CORROBORATION); + if (udpRefused !== null) { + const readings = [...disputed.entries()].map(([peer, port]) => `${peer} on port ${port}`); + log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} at this address does not answer this node over UDP, as heard by ` + + `${disputed.size} peers (${readings.join(', ')}). Installation aborted.`); + decision = { + ok: false, + reason: 'udpUnreachable', + port: udpRefused, + peers: [...disputed.keys()], + readings: Object.fromEntries(disputed), + }; + fluxEventBus.publish('ports:udpUnreachable', decision); + break; + } + } + + if (reportedPort === null) { + // eslint-disable-next-line no-continue + continue; + } + disagreements.set(askingIP, reportedPort); const refused = refusedPort(disagreements, PORT_TEST_CORROBORATION); @@ -1094,7 +1216,11 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // existed, and it is what stops the first nodes to upgrade refusing // everything while the rest of the network catches up. let reason; - if (disagreements.size) { + if (tcpProven) { + // Proven over TCP; over UDP either no peer that probes it was asked, or + // fewer than two heard nothing. + reason = 'udpUnproven'; + } else if (disagreements.size) { reason = ranOutOfObservers ? 'noOtherObserver' : 'singleWitness'; } else if (ranOutOfObservers) { reason = 'noObserver'; @@ -1104,10 +1230,15 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { reason = 'noneAnswered'; } - const witnesses = [...disagreements.keys()]; - const disputed = disagreements.size ? [...disagreements.values()][0] : null; + const evidence = reason === 'udpUnproven' ? udpDisputes(udpDisagreements, udpProven) : disagreements; + const witnesses = [...evidence.keys()]; + const disputed = evidence.size ? [...evidence.values()][0] : null; - if (witnesses.length) { + if (reason === 'udpUnproven') { + log.warn(`checkInstallingAppPortAvailable - proven over TCP; over UDP ${witnesses.length} peer(s) heard nothing ` + + `(${witnesses.join(', ') || 'none'})${sawNoUdpPeer ? ' and some peers asked do not probe UDP' : ''}; ` + + 'proceeding without proof over UDP'); + } else if (witnesses.length) { log.warn(`checkInstallingAppPortAvailable - ${witnesses.length} peer(s) read a port that was not ours ` + `(${witnesses.join(', ')}) and no other Flux node outside this address could be asked; ` + 'proceeding on reachability alone'); @@ -1135,13 +1266,16 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { for (const portToTest of portsToTest) { if (firewallActive) { // eslint-disable-next-line no-await-in-loop - await fluxNetworkHelper.deleteTestPortRule(portToTest); + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'tcp'); + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'udp'); } if (isUPNP) { // eslint-disable-next-line no-await-in-loop await upnpService.removeMapUpnpPort(portToTest, description); } } + udpTestServers.forEach((udpTestServer) => udpTestServer.close()); // Close all test servers and wait for them to finish await Promise.all( beforeAppInstallTestingServers.map((server) => new Promise((resolve) => { @@ -1162,13 +1296,16 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { for (const portToTest of portsToTest) { if (firewallActive) { // eslint-disable-next-line no-await-in-loop - await fluxNetworkHelper.deleteTestPortRule(portToTest).catch((e) => log.error(e)); + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'tcp').catch((e) => log.error(e)); + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'udp').catch((e) => log.error(e)); } if (isUPNP) { // eslint-disable-next-line no-await-in-loop await upnpService.removeMapUpnpPort(portToTest, description).catch((e) => log.error(e)); } } + udpTestServers.forEach((udpTestServer) => udpTestServer.close()); // Close all test servers and wait for them to finish await Promise.all( beforeAppInstallTestingServers.map((server) => new Promise((resolve) => { @@ -1307,6 +1444,9 @@ module.exports = { portsInUse, portsInUseApi, portNotOurs, + udpPortsNotOurs, + udpDisputes, + everyUdpPortProven, refusedPort, siblingHoldingPort, checkInstallingAppPortAvailable, diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index d686e07342..5e5e4613f2 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -12,6 +12,7 @@ const appUninstaller = require('../../ZelBack/src/services/appLifecycle/appUnins const networkStateService = require('../../ZelBack/src/services/networkStateService'); const { EventEmitter } = require('node:events'); const fluxHttpTestServer = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); const { requireMongo } = require('./dbTestHelper'); const appQueryService = require('../../ZelBack/src/services/appQuery/appQueryService'); const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); @@ -1198,6 +1199,8 @@ describe('portManager tests', () => { // one witness to accept, two to refuse - and where the exits that skip them are. describe('checkInstallingAppPortAvailable decides on every way of running out', () => { let port; + let token; + let udpServers; const PEERS = ['10.0.0.1:16127', '10.0.0.2:16127', '10.0.0.3:16127', '10.0.0.4:16127', '10.0.0.5:16127']; @@ -1225,12 +1228,20 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', }); beforeEach(() => { - sinon.stub(fluxHttpTestServer, 'FluxHttpTestServer').callsFake(() => { + token = null; + udpServers = []; + sinon.stub(fluxHttpTestServer, 'FluxHttpTestServer').callsFake((runToken) => { + token = runToken; const server = new EventEmitter(); server.listen = () => { setImmediate(() => server.emit('listening')); return server; }; server.close = (callback) => { if (callback) setImmediate(callback); return server; }; return server; }); + sinon.stub(fluxUdpTestServer, 'FluxUdpTestServer').callsFake(() => { + const server = { listen: sinon.stub().resolves(), allow: sinon.stub(), close: sinon.stub() }; + udpServers.push(server); + return server; + }); sinon.stub(serviceHelper, 'delay').resolves(); sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('1.2.3.4:16127'); sinon.stub(fluxNetworkHelper, 'getFluxNodePublicKey').resolves('04pubkey'); @@ -1256,7 +1267,8 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', }); sinon.stub(axios, 'post').callsFake(async () => { - const answer = answers[attempt]; + // An answer that carries our token is written against the run's token. + const answer = typeof answers[attempt] === 'function' ? answers[attempt](token) : answers[attempt]; if (!answer || answer === UNREACHABLE) throw new Error('connect ECONNREFUSED'); return answer; }); @@ -1289,9 +1301,9 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', await withPeers([UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE]); - sinon.assert.calledOnceWithExactly(allowTest, port); - sinon.assert.calledWith(deleteTest, port); - expect(deleteTest.firstCall.calledAfter(allowTest.firstCall)).to.equal(true); + expect(allowTest.args).to.deep.equal([[port, 'tcp'], [port, 'udp']]); + expect(deleteTest.args).to.deep.equal([[port, 'tcp'], [port, 'udp']]); + expect(deleteTest.firstCall.calledAfter(allowTest.lastCall)).to.equal(true); sinon.assert.notCalled(allowBoth); sinon.assert.notCalled(deleteBoth); }); @@ -1388,4 +1400,188 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', sinon.assert.calledOnceWithExactly(mapUpnpPort, port, 'Flux_App_web'); }); + + // What a peer that probes UDP answers. `udp` is the port's UDP reply: our + // token's reply, null for silence, or absent for a port it could not send to. + const answering = (udp) => (runToken) => ({ + data: { + status: 'success', + data: { + answered: { [port]: `HTTP/1.1 200 OK\r\nX-Flux-Port-Test: ${runToken}\r\n\r\n` }, + udpAnswered: udp === undefined ? {} : { [port]: udp === 'ours' ? fluxUdpTestServer.udpReply(runToken).toString() : udp }, + }, + }, + }); + // A peer on code that reads TCP and does not probe UDP. + const tcpOnly = (runToken) => ({ + data: { status: 'success', data: { answered: { [port]: `X-Flux-Port-Test: ${runToken}` } } }, + }); + + it('proves the ports when one peer reads our token over TCP and hears it over UDP', async () => { + const result = await withPeers([answering('ours')]); + + expect(result).to.deep.equal({ ok: true, reason: 'proven', port: null }); + sinon.assert.calledOnce(axios.post); + }); + + it('asks the peer to probe UDP, and opens a UDP test server on every port', async () => { + await withPeers([answering('ours')]); + + expect(JSON.parse(axios.post.firstCall.args[1]).udp).to.equal(true); + expect(udpServers).to.have.length(1); + sinon.assert.calledWith(udpServers[0].listen, port); + sinon.assert.calledOnce(udpServers[0].close); + }); + + // A UDP reply goes wherever a probe claims to come from; answering anyone + // but the peer being asked would make the port under test a reflector. + it('answers only the peer it is asking, from before it asks', async () => { + await withPeers([answering(null), answering('ours')]); + + expect(udpServers[0].allow.args).to.deep.equal([['10.0.0.1'], ['10.0.0.2']]); + expect(udpServers[0].allow.secondCall.calledBefore(axios.post.secondCall)).to.equal(true); + }); + + it('refuses once two distinct peers hear nothing over UDP', async () => { + const result = await withPeers([answering(null), answering(null), UNREACHABLE, UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'two peers hearing nothing did not refuse').to.equal(false); + expect(result.reason).to.equal('udpUnreachable'); + expect(result.port).to.equal(port); + expect(result.peers).to.deep.equal(['10.0.0.1', '10.0.0.2']); + }); + + it('refuses on a UDP answer that is not our token, as on silence', async () => { + const result = await withPeers([answering('somebody else'), answering(null)]); + + expect(result.ok).to.equal(false); + expect(result.reason).to.equal('udpUnreachable'); + }); + + // UDP loses datagrams. Our token heard by one peer is proof, and another + // peer's silence on the same port is loss. + it('takes our token heard by one peer over silence heard by another', async () => { + const result = await withPeers([answering(null), answering('ours'), answering(null)]); + + expect(result).to.deep.equal({ ok: true, reason: 'proven', port: null }); + }); + + it('proceeds without proof over UDP when one peer alone heard nothing', async () => { + const result = await withPeers([answering(null), UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'refused on a single UDP witness').to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + expect(result.port).to.equal(port); + expect(result.peers).to.deep.equal(['10.0.0.1']); + }); + + // A peer that could not send says nothing about the port. + it('counts a port the peer could not send to as no evidence', async () => { + const result = await withPeers([answering(undefined), answering(undefined), answering(undefined), answering(undefined), answering(undefined)]); + + expect(result.ok).to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + expect(result.peers).to.deep.equal([]); + }); + + // While the fleet carries peers that do not probe UDP, an install goes ahead + // on TCP proof, having asked every peer it may for UDP. + it('proceeds on TCP proof when no peer asked probes UDP', async () => { + const result = await withPeers([tcpOnly, tcpOnly, tcpOnly, tcpOnly, tcpOnly]); + + expect(result.ok).to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + sinon.assert.callCount(axios.post, 5); + }); + + // Our token over TCP is proof; peers that later fail to reach the port over + // TCP cannot outweigh it. + it('lets no later peers that could not reach the port outweigh TCP proof', async () => { + const result = await withPeers([tcpOnly, couldNotReach(port), couldNotReach(port), UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'TCP proof was outweighed').to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + }); + + it('lets no later peers reading something else outweigh TCP proof', async () => { + const result = await withPeers([tcpOnly, answeringSomethingElse(port), answeringSomethingElse(port), UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'TCP proof was outweighed').to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + }); + + // Two ports. The first peer hears nothing on either; the second proves the + // first port and hears nothing on the second. Both peers are silent on the + // second port, and the first peer's silence on the first is withdrawn. + it('corroborates on the port two peers heard nothing on, withdrawing what was proven', async () => { + const other = port + 1; + const two = (udp) => (runToken) => ({ + data: { + status: 'success', + data: { + answered: { [port]: `X-Flux-Port-Test: ${runToken}`, [other]: `X-Flux-Port-Test: ${runToken}` }, + udpAnswered: Object.fromEntries(Object.entries(udp).map(([p, v]) => [p, v === 'ours' ? fluxUdpTestServer.udpReply(runToken).toString() : v])), + }, + }, + }); + let attempt = -1; + sinon.stub(networkStateService, 'getRandomExternalObserver').callsFake(async () => { attempt += 1; return PEERS[attempt] || null; }); + const answers = [two({ [port]: null, [other]: null }), two({ [port]: 'ours', [other]: null })]; + sinon.stub(axios, 'post').callsFake(async () => { + if (!answers[attempt]) throw new Error('connect ECONNREFUSED'); + return answers[attempt](token); + }); + + const result = await portManager.checkInstallingAppPortAvailable([port, other], 'web'); + + expect(result.ok).to.equal(false); + expect(result.reason).to.equal('udpUnreachable'); + expect(result.port).to.equal(other); + expect(result.readings).to.deep.equal({ '10.0.0.1': other, '10.0.0.2': other }); + }); + + it('closes its UDP test servers when the test fails', async () => { + fluxNetworkHelper.getLocalSocketAddress.resolves('1.2.3.4:16127'); + fluxUdpTestServer.FluxUdpTestServer.callsFake(() => { + const server = { listen: sinon.stub().rejects(new Error('EADDRINUSE')), allow: sinon.stub(), close: sinon.stub() }; + udpServers.push(server); + return server; + }); + + const result = await withPeers([answering('ours')]); + + expect(result).to.deep.equal({ ok: false, reason: 'error' }); + sinon.assert.calledOnce(udpServers[0].close); + }); +}); + +describe('udpPortsNotOurs and udpDisputes', () => { + const TOKEN = 'a'.repeat(32); + const ours = fluxUdpTestServer.udpReply(TOKEN).toString(); + + it('adds every port our token came back from to the proven set', () => { + const proven = new Set(); + expect(portManager.udpPortsNotOurs([31000, 31001], { 31000: ours, 31001: ours }, TOKEN, proven)).to.deep.equal([]); + expect([...proven]).to.deep.equal([31000, 31001]); + }); + + it('names every port this peer heard nothing of ours on', () => { + expect(portManager.udpPortsNotOurs([31000, 31001, 31002], { 31000: ours, 31001: null, 31002: 'other' }, TOKEN, new Set())) + .to.deep.equal([31001, 31002]); + }); + + it('reads a port missing from the answers as no evidence', () => { + expect(portManager.udpPortsNotOurs([31000], {}, TOKEN, new Set())).to.deep.equal([]); + }); + + it('skips ports outside the app port range, which the peer does not probe', () => { + const proven = new Set(); + expect(portManager.udpPortsNotOurs([0], { 0: null }, TOKEN, proven)).to.deep.equal([]); + expect(portManager.everyUdpPortProven([0], proven)).to.equal(true); + }); + + it('keeps a peer as a witness only while a port it heard nothing on is unproven', () => { + const disagreements = new Map([['a', [31000, 31001]], ['b', [31000]]]); + expect(portManager.udpDisputes(disagreements, new Set([31000]))).to.deep.equal(new Map([['a', 31001]])); + }); }); diff --git a/tests/unit/portTestEcho.test.js b/tests/unit/portTestEcho.test.js index 9cc1042200..0079db8cdc 100644 --- a/tests/unit/portTestEcho.test.js +++ b/tests/unit/portTestEcho.test.js @@ -5,6 +5,7 @@ const sinon = require('sinon'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); const { FluxHttpTestServer } = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); +const { FluxUdpTestServer } = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const portManager = require('../../ZelBack/src/services/appNetwork/portManager'); const { Privilege } = require('../../ZelBack/src/services/utils/privileges'); @@ -96,6 +97,71 @@ describe('the pre-install port test, end to end', () => { }); }); +// The UDP half, end to end: this node's UDP test server answers a probe with +// the run's secret, the peer probes and hands back what replied, and this node +// compares. +describe('the pre-install port test over UDP, end to end', () => { + const TOKEN = 'f00dcafe'.repeat(4); + // Short enough that silence costs the suite little; the schedule's shape is + // the default's. + const QUICK = { schedule: [0, 50, 150], deadline: 400 }; + let server; + let port; + + beforeEach(async () => { + server = new FluxUdpTestServer(TOKEN); + await server.listen(0, '127.0.0.1'); + ({ port } = server.address()); + }); + + afterEach(() => server.close()); + + it('proves the port when our own test server is what answers', async () => { + server.allow('127.0.0.1'); + + const answer = await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', port, QUICK); + + expect(answer, 'the peer heard nothing back').to.be.a('string'); + const proven = new Set(); + expect(portManager.udpPortsNotOurs([port], { [port]: answer }, TOKEN, proven)).to.deep.equal([]); + expect(proven.has(port)).to.equal(true); + }); + + it('hears nothing from a test server answering another peer', async () => { + server.allow('198.51.100.7'); + + expect(await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', port, QUICK)).to.equal(null); + }); + + it('hears nothing from a port with nothing on it', async () => { + server.close(); + + expect(await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', port, QUICK)).to.equal(null); + }); + + // A datagram lost on the way is the normal case for UDP, not a closed port. + it('is answered by a later datagram when the first is lost', async () => { + const lossy = dgram.createSocket('udp4'); + await new Promise((resolve) => { lossy.bind(0, '127.0.0.1', resolve); }); + const lossyPort = lossy.address().port; + let seen = 0; + lossy.on('message', (message, rinfo) => { + seen += 1; + if (seen > 1) lossy.send(Buffer.from(`FLUXPT ${TOKEN}`), rinfo.port, rinfo.address); + }); + + const answer = await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', lossyPort, QUICK); + lossy.close(); + + expect(answer).to.include(TOKEN); + expect(seen).to.be.at.least(2); + }); + + it('says nothing about a port it could not send to', async () => { + expect(await fluxNetworkHelper.portAnsweredUdp('::1', port, QUICK)).to.equal(undefined); + }); +}); + // The address this node connects to when a peer asks it to test some ports. // // It is NOT taken from the request body. A caller that names the address picks @@ -226,6 +292,91 @@ describe('checkAppAvailability probes the address that asked', () => { ); }); + it('probes each port over UDP when asked, handing back what replied', async () => { + const udpServer = new FluxUdpTestServer(TOKEN); + await udpServer.listen(port, HERE); + udpServer.allow(HERE); + + const answer = await ask( + { + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, udp: true, + }, + HERE, + ); + udpServer.close(); + + expect(answer.status).to.equal('success'); + expect(answer.data.answered[port]).to.include(TOKEN); + expect(answer.data.udpAnswered[port]).to.include(TOKEN); + }); + + // The test server answers over IPv6 too, so TCP passes; a UDP probe is IPv4 + // and cannot be sent there. That says nothing about the port. + it('leaves out a port it could not send a UDP probe to', async () => { + const dualStack = new FluxHttpTestServer(TOKEN); + await new Promise((resolve) => dualStack.listen(0, '::1', resolve)); + const v6Port = dualStack.address().port; + + const answer = await ask( + { + ip: '::1', port: 16127, ports: [v6Port], pubKey: 'k', signature: 's', echo: true, udp: true, + }, + '::1', + ); + await new Promise((resolve) => dualStack.close(resolve)); + + expect(answer.status).to.equal('success'); + expect(answer.data.answered[v6Port]).to.include(TOKEN); + expect(answer.data.udpAnswered).to.deep.equal({}); + }); + + it('probes no UDP for a caller that does not ask', async () => { + const answer = await ask( + { + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, + }, + HERE, + ); + + expect(answer.status).to.equal('success'); + expect(answer.data).to.not.have.property('udpAnswered'); + }); + + it('answers for every port when asked, a port that did not answer reading null', async () => { + const gone = http.createServer(); + await new Promise((resolve) => gone.listen(0, HERE, resolve)); + const closedPort = gone.address().port; + await new Promise((resolve) => gone.close(resolve)); + + const answer = await ask( + { + ip: HERE, port: 16127, ports: [closedPort, port], pubKey: 'k', signature: 's', echo: true, everyPort: true, + }, + HERE, + ); + + expect(answer.status, 'the test ended at the first port that failed').to.equal('success'); + expect(answer.data.answered[closedPort]).to.equal(null); + expect(answer.data.answered[port]).to.include(TOKEN); + }); + + it('ends at the first port that failed for a caller that does not ask for every port', async () => { + const gone = http.createServer(); + await new Promise((resolve) => gone.listen(0, HERE, resolve)); + const closedPort = gone.address().port; + await new Promise((resolve) => gone.close(resolve)); + + const answer = await ask( + { + ip: HERE, port: 16127, ports: [closedPort, port], pubKey: 'k', signature: 's', echo: true, + }, + HERE, + ); + + expect(answer.status).to.equal('error'); + expect(answer.data.message).to.include(`Failed port: ${closedPort}`); + }); + // maxComponents (10) x ports per component (5) in appValidator. Each port // costs a connect timeout and they are tested in sequence. it('refuses more ports than an application could hold', async () => { From 14fc518e7946e359310a94233a03df8ad84ca988 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 08:31:29 +0100 Subject: [PATCH 150/369] feat(ports): the availability check proves its port over TCP and UDP Each pass publishes a token on an HTTP and a UDP test server, maps the port for both protocols, opens a test rule for each, and asks the peer to read the port and probe it over UDP. A port the peer read as something other than this node fails, as a port nothing answered does: an application placed on it here would not be reached. A port an application at this public address holds, by the network's records or by a node sharing the address, is skipped untested. Our token over UDP proves the port. One peer's silence is tested again by another peer on the next pass, for up to three passes; two peers' silence is a confirmed UDP failure. It is reported as the `udp` check in /flux/health, degraded, and counts toward this check's DOS only when the signed policy document `udpportdos` is `true`. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/availabilityChecker.js | 160 +++++++++++++- ZelBack/src/services/fluxNetworkHelper.js | 20 ++ ZelBack/src/services/nodeHealthService.js | 7 + tests/unit/availabilityChecker.test.js | 202 +++++++++++++++++- tests/unit/nodeHealthService.test.js | 21 +- 5 files changed, 394 insertions(+), 16 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index 022e8372b0..02ddf9add0 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -1,4 +1,5 @@ // Availability Checker - Checks apps availability by testing ports from external nodes +const crypto = require('node:crypto'); const axios = require('axios'); const config = require('config'); const serviceHelper = require('../serviceHelper'); @@ -8,6 +9,10 @@ const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const upnpService = require('../upnpService'); const networkStateService = require('../networkStateService'); const fluxHttpTestServer = require('../utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../utils/fluxUdpTestServer'); +const portManager = require('../appNetwork/portManager'); +const registryManager = require('../appDatabase/registryManager'); +const policyStore = require('../policyStore'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -19,6 +24,8 @@ async function handleTestShutdown(testingPort, testHttpServer, isArcane, options const skipUpnp = options.skipUpnp || false; const skipHttpServer = options.skipHttpServer || false; + if (options.udpTestServer) options.udpTestServer.close(); + const updateFirewall = skipFirewall ? false : isArcane @@ -26,7 +33,10 @@ async function handleTestShutdown(testingPort, testHttpServer, isArcane, options if (updateFirewall) { await fluxNetworkHelper - .deleteTestPortRule(testingPort) + .deleteTestPortRule(testingPort, 'tcp') + .catch((e) => log.error(e)); + await fluxNetworkHelper + .deleteTestPortRule(testingPort, 'udp') .catch((e) => log.error(e)); } @@ -97,6 +107,58 @@ function releaseOwnDos(dosState) { dosState.dosStateValue = 0; } +/** + * The signed policy document that lets a port found closed over UDP count + * toward this check's DOS. Absent or anything but `true`, a confirmed UDP + * failure is reported in the node's health and counts nothing. + */ +const UDP_DOS_DOCUMENT = 'udpportdos'; + +/** + * Whether a port found closed over UDP counts toward this check's DOS. + * @returns {boolean} + */ +function udpDosEnabled() { + return policyStore.getDocument(UDP_DOS_DOCUMENT) === true; +} + +/** + * How many passes a port one peer heard nothing on over UDP is tested again, + * looking for a second peer that probes UDP, before that peer's silence is + * dropped unconfirmed. + */ +const UDP_RETESTS = 3; + +/** + * Whether an application the network lists at this node's public address holds + * a port. Another node behind the same address serving that port is not a fault + * of this node's, so the port is not one to test here. + * @param {number} port + * @param {string} localSocketAddress + * @returns {Promise} + */ +async function portHeldAtThisAddress(port, localSocketAddress) { + const appsAtThisAddress = await registryManager.getRunningAppIpList(extractIp(localSocketAddress)); + const held = await portManager.assignedPortsGlobalApps(appsAtThisAddress.map((app) => app.name)); + if (held.some((app) => app.ports.includes(port))) return true; + return (await portManager.siblingHoldingPort([port], localSocketAddress)) !== null; +} + +/** + * What one peer's UDP answer for the port under test says. + * @param {object|undefined} udpAnswered - from the peer + * @param {number} port + * @param {string} token + * @returns {'proven'|'silent'|'unknown'} unknown: the peer does not probe UDP, + * or could not send to the port + */ +function udpVerdict(udpAnswered, port, token) { + if (!udpAnswered || typeof udpAnswered !== 'object') return 'unknown'; + const reply = udpAnswered[port] !== undefined ? udpAnswered[port] : udpAnswered[String(port)]; + if (reply === undefined) return 'unknown'; + return typeof reply === 'string' && reply.includes(token) ? 'proven' : 'silent'; +} + let wakeNextPass = null; /** @@ -154,7 +216,11 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const isUpnp = upnpService.isUPNP(); let routerRefused = false; - const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(); + // One secret for this pass, published by both test servers and never sent to + // the peer: only our own test servers can produce it. + const portTestToken = crypto.randomBytes(16).toString('hex'); + const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(portTestToken); + let udpTestServer = null; const setNextPort = () => { if (dosState.originalPortFailed && dosState.testingPort > dosState.originalPortFailed) { @@ -282,10 +348,24 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin return; } + if (await portHeldAtThisAddress(dosState.testingPort, localSocketAddress)) { + log.info(`checkMyAppsAvailability - Skipped checking ${dosState.testingPort} - held by an application at this address`); + setNextPort(); + await waitForNextPass(timeouts.failure); + setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); + return; + } + + // A port one peer heard nothing on over UDP is tested again by another. + const udpSuspect = dosState.udpSuspect?.port === dosState.testingPort ? dosState.udpSuspect : null; + // An external observer: this asks a peer whether it can reach US, and a Flux // node sharing our public address cannot answer that. Null when there is no // such node, which the retry below already handles. - const remoteSocketAddress = await networkStateService.getRandomExternalObserver(localSocketAddress); + const remoteSocketAddress = await networkStateService.getRandomExternalObserver( + localSocketAddress, + { exclude: udpSuspect ? [udpSuspect.peer] : [] }, + ); if (!remoteSocketAddress) { await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); @@ -300,12 +380,12 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const firewallActive = isArcane ? true : await fluxNetworkHelper.isFirewallActive(); if (firewallActive) { - await fluxNetworkHelper.allowTestPort(dosState.testingPort); + await fluxNetworkHelper.allowTestPort(dosState.testingPort, 'tcp'); + await fluxNetworkHelper.allowTestPort(dosState.testingPort, 'udp'); } if (isUpnp) { - // the test server is HTTP: TCP is all it needs - const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); + const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App'); routerRefused = routerRefusedSinceLastPass(dosState); if (routerRefused) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); if (!upnpMapResult.mapped) { @@ -335,14 +415,20 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin testHttpServer.listen(dosState.testingPort); }); - const error = await listening.catch((err) => err); + let error = await listening.catch((err) => err); + const httpListening = !error; + if (httpListening) { + udpTestServer = new fluxUdpTestServer.FluxUdpTestServer(portTestToken); + error = await udpTestServer.listen(dosState.testingPort).then(() => null, (err) => err.message); + } if (error) { log.warn(`Unable to listen on port: ${dosState.testingPort}. Error: ${error}`); setNextPort(); await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipUpnp: !isUpnp, - skipHttpServer: true, + skipHttpServer: !httpListening, + udpTestServer, }); await waitForNextPass(timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); @@ -369,12 +455,17 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin appname: 'appPortsTest', ports: [dosState.testingPort], pubKey: signer.pubKey, + // Read the port back rather than merely reach it, and probe it over UDP. + echo: true, + udp: true, }; const signature = signer.sign(JSON.stringify(data)); if (!signature) throw new Error('checkMyAppsAvailability - the port test could not be signed'); data.signature = signature; + udpTestServer.allow(remoteIp); + const resMyAppAvailability = await axios .post(`http://${remoteIp}:${remotePort}/flux/checkappavailability`, JSON.stringify(data), axiosConfig) .catch(() => { @@ -388,6 +479,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipUpnp: !isUpnp, + udpTestServer, }); if (!resMyAppAvailability) { @@ -410,7 +502,49 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin return; } - const portTestFailed = responseStatus === 'error'; + const { answered, udpAnswered } = resMyAppAvailability.data.data || {}; + let portTestFailed = responseStatus === 'error'; + + // A peer that read the port: only our token says this node is what + // answered. Anything else holding the port leaves it as unusable here as + // nothing answering. A peer that does not read ports is taken at its word. + if (!portTestFailed && answered && portManager.portNotOurs([dosState.testingPort], answered, portTestToken) !== null) { + log.warn(`checkMyAppsAvailability - Port ${dosState.testingPort} answered ${remoteIp}:${remotePort} as something other than this node`); + portTestFailed = true; + } + + // Over UDP, our token heard by one peer proves the port; silence heard by + // one peer is tested again by another, and counts once two have heard it. + let retestUdp = false; + const udp = portTestFailed ? 'unknown' : udpVerdict(udpAnswered, dosState.testingPort, portTestToken); + if (udp === 'proven') { + // eslint-disable-next-line no-param-reassign + dosState.udpSuspect = null; + fluxNetworkHelper.setUdpReachability({ reachable: true }); + } else if (udp === 'silent' && !udpSuspect) { + log.warn(`checkMyAppsAvailability - ${remoteIp}:${remotePort} heard nothing of this node on port ${dosState.testingPort} over UDP; asking another peer`); + // eslint-disable-next-line no-param-reassign + dosState.udpSuspect = { port: dosState.testingPort, peer: remoteSocketAddress, retests: 0 }; + // eslint-disable-next-line no-param-reassign + dosState.nextTestingPort = dosState.testingPort; + await waitForNextPass(timeouts.failure); + setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); + return; + } else if (udp === 'silent') { + const peers = [extractIp(udpSuspect.peer), remoteIp]; + log.error(`checkMyAppsAvailability - Port ${dosState.testingPort} does not answer this node over UDP, as heard by ${peers.join(' and ')}` + + `${udpDosEnabled() ? '' : '; reported in health, not counted'}`); + // eslint-disable-next-line no-param-reassign + dosState.udpSuspect = null; + fluxNetworkHelper.setUdpReachability({ reachable: false, port: dosState.testingPort, peers }); + if (udpDosEnabled()) portTestFailed = true; + } else if (udpSuspect && !portTestFailed) { + udpSuspect.retests += 1; + retestUdp = udpSuspect.retests < UDP_RETESTS; + // eslint-disable-next-line no-param-reassign + if (!retestUdp) dosState.udpSuspect = null; + } + let waitMs = 0; if (portTestFailed && portsNotWorking.size < thresholds.portsHighEdge) { @@ -462,6 +596,12 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin log.error(`checkMyAppsAvailability - Count: ${portsNotWorking.size}. portsNotWorking: ${JSON.stringify(Array.from(portsNotWorking))}`); } + if (retestUdp) { + // eslint-disable-next-line no-param-reassign + dosState.nextTestingPort = dosState.testingPort; + waitMs = Math.min(waitMs, timeouts.failure); + } + // A router that refused this node a mapping or has yet to remove one is // looked at again soon, so the pass that finds it working again comes // soon; a refusal during this pass gave no wake, the check not waiting. @@ -474,7 +614,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { - await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp }); + await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp, udpTestServer }); log.error(`checkMyAppsAvailability - Error: ${error}`); await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 31f4bcb74a..8b40b1959e 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -1032,6 +1032,24 @@ function getDosMessage() { return getStickyDosMessage() || dosMessage; } +// Whether this node's ports answer over UDP, as the availability check last +// established it with a peer's proof or two peers' silence; null until it has. +let udpReachability = null; + +/** + * @param {{reachable: boolean, port?: number, peers?: string[]}|null} value + */ +function setUdpReachability(value) { + udpReachability = value; +} + +/** + * @returns {{reachable: boolean, port?: number, peers?: string[]}|null} + */ +function getUdpReachability() { + return udpReachability; +} + /** * Take this node out of service for as long as one owner says so. Idempotent * per owner. @@ -3656,6 +3674,8 @@ module.exports = { setLocalSocketAddress, getDosMessage, setDosMessage, + getUdpReachability, + setUdpReachability, setDosStateValue, getDosStateValue, isNodeDos, diff --git a/ZelBack/src/services/nodeHealthService.js b/ZelBack/src/services/nodeHealthService.js index b333917565..3e7dab9e5c 100644 --- a/ZelBack/src/services/nodeHealthService.js +++ b/ZelBack/src/services/nodeHealthService.js @@ -195,6 +195,13 @@ async function checkNodeFitness() { // that has stopped answering, and daemonHealthMonitor acts on one that stays so. checks.daemon = daemonServiceMiscRpcs.isDaemonSynced().data.synced ? CHECK.OK : CHECK.DEGRADED; + // udp: whether this node's ports answer over UDP, as the availability check + // last established it. Reported, never refused on here: whether a port closed + // over UDP takes the node out of service is that check's to decide. + const udp = fluxNetworkHelper.getUdpReachability(); + if (udp === null) checks.udp = CHECK.UNMEASURED; + else checks.udp = udp.reachable ? CHECK.OK : CHECK.DEGRADED; + // hardware: the node must still meet its tier requirements const hardware = await confirmNodeTierHardware(); if (hardware.result === CHECK.FAIL) { diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 2a0c6cc327..6d9138896a 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -14,6 +14,10 @@ const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/ const upnpService = require('../../ZelBack/src/services/upnpService'); const networkStateService = require('../../ZelBack/src/services/networkStateService'); const fluxHttpTestServer = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); +const portManager = require('../../ZelBack/src/services/appNetwork/portManager'); +const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); +const policyStore = require('../../ZelBack/src/services/policyStore'); const signer = { pubKey: 'pubkey', sign: () => 'signature' }; // the check with a node that can sign its port test @@ -29,6 +33,8 @@ describe('availabilityChecker tests', () => { let isArcane; let delayStub; let setImmediateStub; + let udpServers; + let udpToken; beforeEach(() => { mockInstalledAppsFn = sinon.stub(); @@ -48,10 +54,25 @@ describe('availabilityChecker tests', () => { delayStub = sinon.stub(serviceHelper, 'delay').resolves(); // Stub setImmediate to prevent infinite recursion setImmediateStub = sinon.stub(global, 'setImmediate'); + + // No application at this address holds a port, unless a test says so. + sinon.stub(registryManager, 'getRunningAppIpList').resolves([]); + sinon.stub(portManager, 'assignedPortsGlobalApps').resolves([]); + sinon.stub(portManager, 'siblingHoldingPort').resolves(null); + + udpServers = []; + udpToken = null; + sinon.stub(fluxUdpTestServer, 'FluxUdpTestServer').callsFake((token) => { + udpToken = token; + const server = { listen: sinon.stub().resolves(), allow: sinon.stub(), close: sinon.stub() }; + udpServers.push(server); + return server; + }); }); afterEach(() => { sinon.restore(); + fluxNetworkHelper.setUdpReachability(null); }); describe('checkMyAppsAvailability tests', () => { @@ -405,10 +426,10 @@ describe('availabilityChecker tests', () => { expect(mockDosState.portFailureScore ?? 0).to.equal(0); }); - it('maps the port under test over TCP alone', async () => { + it('maps the port under test over TCP and UDP', async () => { await pass(); - sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); + sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App'); }); it('makes no test mapping, opens no port, and counts a failed check while the router has yet to remove a mapping of this node', async () => { @@ -532,8 +553,8 @@ describe('availabilityChecker tests', () => { await pass(); - sinon.assert.calledOnceWithExactly(fluxNetworkHelper.allowTestPort, mockDosState.testingPort); - sinon.assert.calledWith(fluxNetworkHelper.deleteTestPortRule, mockDosState.testingPort); + expect(fluxNetworkHelper.allowTestPort.args).to.deep.equal([[mockDosState.testingPort, 'tcp'], [mockDosState.testingPort, 'udp']]); + expect(fluxNetworkHelper.deleteTestPortRule.args).to.deep.equal([[mockDosState.testingPort, 'tcp'], [mockDosState.testingPort, 'udp']]); sinon.assert.notCalled(allowBoth); sinon.assert.notCalled(deleteBoth); }); @@ -610,6 +631,179 @@ describe('availabilityChecker tests', () => { }); }); + describe('proof over TCP and UDP', () => { + const PEERS = ['192.168.1.200:16127', '192.168.1.201:16127', '192.168.1.202:16127', '192.168.1.203:16127', '192.168.1.204:16127']; + let observer; + let portTestStub; + + // What a peer that reads the port and probes UDP answers: `tcp` and `udp` + // are 'ours', another application's bytes, or null for silence; `udp` + // undefined is a peer that does not probe UDP. + const reading = (tcp, udp) => async (url, body) => { + const { ports: [port] } = JSON.parse(body); + const value = (v, ours) => (v === 'ours' ? ours : v); + const data = { message: 'Ports reachable', answered: { [port]: value(tcp, `X-Flux-Port-Test: ${udpToken}`) } }; + if (udp !== undefined) data.udpAnswered = { [port]: value(udp, fluxUdpTestServer.udpReply(udpToken).toString()) }; + return { data: { status: 'success', data } }; + }; + + beforeEach(() => { + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); + sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); + sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); + mockInstalledAppsFn.resolves({ status: 'success', data: [] }); + sinon.stub(upnpService, 'isUPNP').returns(false); + sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); + observer = -1; + sinon.stub(networkStateService, 'getRandomExternalObserver').callsFake(async () => { + observer += 1; + return PEERS[observer]; + }); + sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(false); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { + this.emit('listening'); + return this; + }); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'close').callsFake((callback) => callback && callback()); + portTestStub = sinon.stub(axios, 'post').callsFake(reading('ours', 'ours')); + }); + + const pass = () => signingAvailabilityChecker.checkMyAppsAvailability( + mockInstalledAppsFn, + mockDosState, + mockPortsNotWorking, + mockFailedNodesCache, + isArcane, + ); + + it('asks the peer to read the port and probe it over UDP, answering only that peer', async () => { + await pass(); + + const body = JSON.parse(portTestStub.firstCall.args[1]); + expect(body.echo).to.equal(true); + expect(body.udp).to.equal(true); + sinon.assert.calledOnceWithExactly(udpServers[0].listen, mockDosState.testingPort); + sinon.assert.calledOnceWithExactly(udpServers[0].allow, '192.168.1.200'); + expect(udpServers[0].allow.calledBefore(portTestStub)).to.equal(true); + sinon.assert.calledOnce(udpServers[0].close); + }); + + it('passes, and reports UDP reachable, on our token over TCP and UDP', async () => { + await pass(); + + expect(mockPortsNotWorking.size).to.equal(0); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + sinon.assert.calledWith(delayStub.lastCall, 3_600_000); + }); + + // Whatever holds the port, this node cannot serve an application on it. + it('counts a port that answered as something other than this node as failed', async () => { + portTestStub.callsFake(reading('somebody else entirely', undefined)); + + await pass(); + + expect(mockPortsNotWorking.has(mockDosState.testingPort)).to.equal(true); + }); + + it('takes a peer that does not read ports at its word', async () => { + portTestStub.resolves({ data: { status: 'success', data: { message: 'Ports reachable' } } }); + + await pass(); + + expect(mockPortsNotWorking.size).to.equal(0); + expect(fluxNetworkHelper.getUdpReachability()).to.equal(null); + }); + + it('skips a port an application at this address holds, testing nothing', async () => { + sinon.stub(Math, 'random').returns(0.5); + const { fluxapps: { portMin, portMax } } = config; + const port = Math.floor(0.5 * (portMax - portMin) + portMin); + registryManager.getRunningAppIpList.resolves([{ name: 'neighbour' }]); + portManager.assignedPortsGlobalApps.resolves([{ name: 'neighbour', ports: [port] }]); + + await pass(); + + sinon.assert.calledWith(portManager.assignedPortsGlobalApps, ['neighbour']); + sinon.assert.notCalled(portTestStub); + expect(mockPortsNotWorking.size).to.equal(0); + }); + + it('skips a port a node sharing this address says it holds, testing nothing', async () => { + portManager.siblingHoldingPort.resolves({ address: '192.168.1.100:16137', port: 1 }); + + await pass(); + + sinon.assert.notCalled(portTestStub); + }); + + // UDP loses datagrams: one peer's silence is tested again by another. + it('counts nothing on one peer\'s silence over UDP, and tests the port again with another peer', async () => { + portTestStub.callsFake(reading('ours', null)); + + await pass(); + const port = mockDosState.testingPort; + + expect(mockPortsNotWorking.size).to.equal(0); + expect(mockDosState.nextTestingPort).to.equal(port); + sinon.assert.calledWith(delayStub.lastCall, 15_000); + expect(fluxNetworkHelper.getUdpReachability()).to.equal(null); + + await pass(); + + expect(mockDosState.testingPort).to.equal(port); + sinon.assert.calledWith(networkStateService.getRandomExternalObserver.secondCall, '192.168.1.100:16127', { exclude: [PEERS[0]] }); + }); + + it('takes our token heard by a second peer over the first peer\'s silence', async () => { + portTestStub.onFirstCall().callsFake(reading('ours', null)); + + await pass(); + await pass(); + + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + expect(mockDosState.udpSuspect).to.equal(null); + }); + + it('reports UDP unreachable, and counts nothing, when two peers hear nothing and policy has not switched the DOS on', async () => { + portTestStub.callsFake(reading('ours', null)); + + await pass(); + await pass(); + + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ + reachable: false, port: mockDosState.testingPort, peers: ['192.168.1.200', '192.168.1.201'], + }); + expect(mockPortsNotWorking.size).to.equal(0); + }); + + it('counts the port failed when two peers hear nothing and policy has switched the DOS on', async () => { + sinon.stub(policyStore, 'getDocument').withArgs('udpportdos').returns(true); + portTestStub.callsFake(reading('ours', null)); + + await pass(); + await pass(); + + expect(mockPortsNotWorking.has(mockDosState.testingPort)).to.equal(true); + expect(fluxNetworkHelper.getUdpReachability().reachable).to.equal(false); + }); + + it('drops one peer\'s silence unconfirmed after as many passes as find no peer that probes UDP', async () => { + portTestStub.onFirstCall().callsFake(reading('ours', null)); + portTestStub.callsFake(reading('ours', undefined)); + + await pass(); + const port = mockDosState.testingPort; + await pass(); + await pass(); + expect(mockDosState.nextTestingPort, 'stopped retesting early').to.equal(port); + await pass(); + + expect(mockDosState.udpSuspect).to.equal(null); + expect(mockDosState.nextTestingPort).to.equal(null); + expect(fluxNetworkHelper.getUdpReachability()).to.equal(null); + }); + }); + it('should handle errors gracefully and retry', async () => { sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').throws(new Error('Service error')); diff --git a/tests/unit/nodeHealthService.test.js b/tests/unit/nodeHealthService.test.js index 82dd97e45a..6f5d800a56 100644 --- a/tests/unit/nodeHealthService.test.js +++ b/tests/unit/nodeHealthService.test.js @@ -390,6 +390,23 @@ describe('nodeHealthService tests', () => { }); }); + // A port closed over UDP is the availability check's to act on; health + // reports it and refuses nobody for it. + it('reports UDP as unmeasured, ok or degraded, and stays fit on each', async () => { + healthyHardware(); + const readings = [[null, 'unmeasured'], [{ reachable: true }, 'ok'], [{ reachable: false, port: 31000, peers: ['a', 'b'] }, 'degraded']]; + const getUdp = sinon.stub(fluxNetworkHelper, 'getUdpReachability'); + + // eslint-disable-next-line no-restricted-syntax + for (const [reading, check] of readings) { + getUdp.returns(reading); + // eslint-disable-next-line no-await-in-loop + const fitness = await nodeHealthService.checkNodeFitness(); + expect(fitness.ok, `unfit on UDP ${check}`).to.equal(true); + expect(fitness.checks.udp).to.equal(check); + } + }); + it('checkNodeFitness returns ok with the checks when the node is fit', async () => { healthyHardware(); @@ -416,7 +433,7 @@ describe('nodeHealthService tests', () => { message: 'Node Total Ram (1) below new Cumulus requirements', failed: 'hardware', checks: { - db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'ok', hardware: 'fail', + db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'ok', udp: 'unmeasured', hardware: 'fail', }, }, }); @@ -513,7 +530,7 @@ describe('nodeHealthService tests', () => { sinon.assert.calledOnceWithExactly(res.json, { status: 'success', data: { - db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'degraded', hardware: 'unmeasured', dos: 'ok', appsDos: 'ok', + db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'degraded', udp: 'unmeasured', hardware: 'unmeasured', dos: 'ok', appsDos: 'ok', }, }); }); From 0764479f51ae495e752251691c5d1cdaee3cfd3f Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 08:38:34 +0100 Subject: [PATCH 151/369] fix(registry): the apps at an IP are those stored at exactly that IP getRunningAppIpList matched every location whose address began with the IP with its dots read as a pattern, so 1.2.3.4 also took the apps of 1.2.3.45. The spawner and the availability check read it as the apps holding ports at this node's public address. storedAtIp builds the condition once for both queries that read every stored address at an IP: the escaped IP and the colon after it, every location and move being stored in ip:port form. Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 28 ++++++++++--- tests/unit/registryManager.test.js | 42 ++++++++++++++++--- 2 files changed, 58 insertions(+), 12 deletions(-) diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index b1ef43256a..d64feddbb6 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -242,18 +242,32 @@ async function addressMovesTo(address) { return moves; } +/** + * The query condition for every stored address at an IP, whatever its port. + * + * Every location and every move is stored under its node's address in ip:port + * form, so the IP and the colon after it match exactly the addresses at that + * IP: never one that only begins with it. + * @param {string} ip An IP, or a socket address whose IP is meant. + * @returns {RegExp|null} null when there is no IP + */ +function storedAtIp(ip) { + const bare = extractIp(ip); + if (!bare) return null; + return new RegExp(`^${bare.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}:`); +} + /** * The socket addresses at `ip` that a recorded move leaves. * @param {string} ip An IP, or a socket address whose IP is meant. * @returns {Promise} */ async function addressesMovedFromIp(ip) { - const bare = extractIp(ip); - if (!bare) return []; + const atIp = storedAtIp(ip); + if (!atIp) return []; const collection = dbHelper.databaseConnection().db(config.database.appsglobal.database).collection(globalAppStateEvents); - const escaped = bare.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const events = await collection.find( - { ip: new RegExp(`^${escaped}:`), type: 'ipchanged', expireAt: { $gt: new Date() } }, + { ip: atIp, type: 'ipchanged', expireAt: { $gt: new Date() } }, { projection: { _id: 0, ip: 1 } }, ).toArray(); return [...new Set(events.map((event) => event.ip))]; @@ -1688,13 +1702,15 @@ async function getRunningApps() { /** * To get all apps running on a specific IP address. Returns all apps running on this ip - * @param {string} ip IP address to check + * @param {string} ip An IP, or a socket address whose IP is meant. * @returns {Promise} Array of apps running on the specified IP */ async function getRunningAppIpList(ip) { + const atIp = storedAtIp(ip); + if (!atIp) return []; const dbopen = dbHelper.databaseConnection(); const database = dbopen.db(config.database.appsglobal.database); - const query = { ip: new RegExp(`^${ip}`) }; + const query = { ip: atIp }; const projection = { projection: { _id: 0, diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 1b9773af46..873b4abf9b 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1666,6 +1666,21 @@ describe('registryManager tests', () => { ip: '192.168.1.2:16127', hash: 'hash3', }, + { + name: 'BeginsWithTheIp', + ip: '192.168.1.10:16127', + hash: 'hash4', + }, + { + name: 'MatchesOnlyAsAPattern', + ip: '192x168x1x1:16127', + hash: 'hash5', + }, + { + name: 'OtherPort', + ip: '192.168.1.1:16137', + hash: 'hash6', + }, ]; try { @@ -1676,14 +1691,29 @@ describe('registryManager tests', () => { await dbHelper.insertManyToDatabase(database, collection, testLocations); }); - it('should return apps running on specific IP', async () => { + it('should return apps running on specific IP, at every port', async () => { const result = await registryManager.getRunningAppIpList('192.168.1.1'); - expect(result).to.be.an('array'); - expect(result.length).to.equal(2); - result.forEach((app) => { - expect(app.ip).to.include('192.168.1.1'); - }); + expect(result.map((app) => app.name).sort()).to.deep.equal(['App1', 'App2', 'OtherPort']); + }); + + // An address that only begins with the IP, or matches it with its dots read + // as a pattern, is another node. + it('returns no app at an address that only begins with the IP or matches it as a pattern', async () => { + const names = (await registryManager.getRunningAppIpList('192.168.1.1')).map((app) => app.name); + + expect(names).to.not.include('BeginsWithTheIp'); + expect(names).to.not.include('MatchesOnlyAsAPattern'); + }); + + it('reads a socket address as its IP', async () => { + const result = await registryManager.getRunningAppIpList('192.168.1.1:16137'); + + expect(result.map((app) => app.name).sort()).to.deep.equal(['App1', 'App2', 'OtherPort']); + }); + + it('returns nothing for no address', async () => { + expect(await registryManager.getRunningAppIpList('')).to.deep.equal([]); }); it('should return empty array for IP with no apps', async () => { From e0c4601f09ef7d80f1b6edcc6dbdd2cc6f50b742 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 08:43:32 +0100 Subject: [PATCH 152/369] test(harness): 3004 proves a port over UDP, refuses one two peers hear nothing on Two network shapes put the fault in a node's own namespace: udp-blocked drops every UDP datagram starting a flow, udp-first-lost drops the first from each source and lets every later one through. On a fleet of nodes on this branch, the open node and the first-lost node install with their ports proven, the blocked node is refused with two distinct peers named, and its availability check reports udp degraded in /flux/health while it stays in service. On a fleet whose peers do not probe UDP, a blocked node installs on TCP proof. The pre-install test publishes ports:proven with the ports and the peer that proved them, beside ports:notOurs and ports:unproven. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appNetwork/portManager.js | 1 + test-infra/network-shapes.sh | 14 ++ test-infra/runner/framework/node-client.js | 2 + .../tests/3004-a-port-is-proven-over-udp.js | 192 ++++++++++++++++++ 4 files changed, 209 insertions(+) create mode 100644 test-infra/runner/tests/3004-a-port-is-proven-over-udp.js diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index b6f2a4a4d4..03b7f4aa06 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -1148,6 +1148,7 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { if (tcpProven && everyUdpPortProven(portsToTest, udpProven)) { decision = { ok: true, reason: 'proven', port: null }; + fluxEventBus.publish('ports:proven', { ...decision, ports: portsToTest, peer: askingIP }); break; } } else { diff --git a/test-infra/network-shapes.sh b/test-infra/network-shapes.sh index 489beb22ee..82b8b98bb3 100755 --- a/test-infra/network-shapes.sh +++ b/test-infra/network-shapes.sh @@ -50,6 +50,13 @@ # direct-nameserver the node's resolv.conf names the fleet's resolver itself, # as a host that lists a public resolver does, rather than # Docker's embedded server in front of it. +# udp-blocked no UDP datagram starting a flow reaches the node, as behind +# a router that forwards TCP alone; replies to the node's own +# datagrams (DNS among them) still arrive. +# udp-first-lost the first UDP datagram from each source starting a flow is +# dropped, and every later one within 30 s arrives: loss that +# a probe sending once never survives and one that sends +# again always does. set -euo pipefail shape="$1" @@ -156,6 +163,13 @@ case "$shape" in : "${FLUX_E2E_RESOLVER:?}" printf 'nameserver %s\n' "$FLUX_E2E_RESOLVER" > /etc/resolv.conf ;; + udp-blocked) + iptables -A INPUT -p udp -m conntrack --ctstate NEW -j DROP + ;; + udp-first-lost) + iptables -A INPUT -p udp -m conntrack --ctstate NEW -m recent --name udpseen --rcheck --seconds 30 -j ACCEPT + iptables -A INPUT -p udp -m conntrack --ctstate NEW -m recent --name udpseen --set -j DROP + ;; *) echo "network-shapes: unknown shape '$shape'" >&2 exit 1 diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index d90e705cb2..62f509a296 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -231,6 +231,8 @@ export function nodeClient(nodeNum) { // it. Nothing existing carries that. 'ports:notOurs', 'ports:unproven', + 'ports:proven', + 'ports:udpUnreachable', 'discovery:started', 'peers:added', 'peers:accepting', diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js new file mode 100644 index 0000000000..61ff346b0c --- /dev/null +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -0,0 +1,192 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { getSubnetConfig, REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { waitFor, waitForAppInstalled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Every app port is published for TCP and UDP, so a node proves both before it +// installs an app, and its availability check proves both on a port of its own. +// +// UDP has no handshake: silence is all a closed port and a lost datagram say. A +// peer probes each port and hands back what replied; only the node's own token +// coming back proves a port, and a refusal needs two distinct peers that heard +// nothing. The shapes put the fault in the node's own namespace: +// +// udp-blocked no datagram starting a flow arrives - a router forwarding TCP +// alone. Refused, and the availability check reports it. +// udp-first-lost the first datagram from each source is lost, every later one +// arrives. A probe that sends once never gets through; one that +// sends again always does. Proven. +// +// The peers in the first fleet are real nodes on this branch. The second fleet's +// peers are stubs that read TCP and do not probe UDP - the state of the network +// while it updates - and a node there must install on TCP proof alone. + +const subnet = getSubnetConfig(); +const socketAddressOf = (index) => `${subnet.nodeIp(index + 1)}:16127`; + +// The availability check's waits, compressed: a pass after a failure comes a +// second later, and a confirmed failure takes two passes. +const APPS_PORT_CHECK = { + timeoutsMs: { + default: 10 * 60 * 1000, error: 5000, failure: 1000, dos: 5 * 60 * 1000, appError: 5000, + }, + thresholds: { dos: 100, portsHighEdge: 3, portsLowEdge: 1 }, +}; + +// Enough attempts that a node always reaches two peers that answer. +const PORT_TEST = { portTestMaxAttempts: 3 }; + +const appPinnedTo = async (name, port, index) => { + await pushImage(name, 'v1'); + return buildSeedableApp({ + name, + instances: 1, + nodes: [socketAddressOf(index)], + compose: [{ + name, + description: 'one port, proven before install', + repotag: `${REGISTRY_REPO_HOST}/${name}:v1`, + ports: [port], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); +}; + +const udpCheck = async (client) => (await client.get('/flux/health').catch(() => null))?.data?.udp; + +describe('a port is proven over UDP', function () { + const OPEN = 0; + const BLOCKED = 1; + const FIRST_LOST = 2; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: { [BLOCKED]: 'udp-blocked', [FIRST_LOST]: 'udp-first-lost' }, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + nodeConfigOverrides: { + [OPEN]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } }, + [BLOCKED]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('installs on a node whose port answers over TCP and UDP, proven by one peer', async function () { + this.timeout(420000); + const after = env.clients[OPEN].getLastEventId(); + const app = await appPinnedTo('udpopenapp', 31201, OPEN); + await seedSpawnerApp(env, app); + + const proven = await env.clients[OPEN].waitForEvent('ports:proven', (d) => d.ports.includes(31201), 420000, { afterId: after }); + expect(proven.data.reason).to.equal('proven'); + await waitForAppInstalled(env.clients[OPEN], 'udpopenapp', 300000, { afterId: after }); + }); + + it('refuses a port two peers hear nothing on over UDP', async function () { + this.timeout(420000); + const after = env.clients[BLOCKED].getLastEventId(); + const app = await appPinnedTo('udpblockedapp', 31202, BLOCKED); + await seedSpawnerApp(env, app); + + const refused = await env.clients[BLOCKED].waitForEvent('ports:udpUnreachable', (d) => d.port === 31202, 420000, { afterId: after }); + expect(refused.data.reason).to.equal('udpUnreachable'); + expect(refused.data.peers).to.have.length(2); + expect(new Set(refused.data.peers).size).to.equal(2); + await env.clients[BLOCKED].waitForEvent( + 'spawner:deferred', + (d) => d.appName === 'udpblockedapp' && d.reason === 'ports_not_available', + 60000, + { afterId: after }, + ); + const installed = await env.clients[BLOCKED].getInstalledApps(); + expect(installed.data.map((a) => a.name)).to.not.include('udpblockedapp'); + }); + + it('proves a port whose first datagram from each peer is lost', async function () { + this.timeout(420000); + const after = env.clients[FIRST_LOST].getLastEventId(); + const app = await appPinnedTo('udplossyapp', 31203, FIRST_LOST); + await seedSpawnerApp(env, app); + + const proven = await env.clients[FIRST_LOST].waitForEvent('ports:proven', (d) => d.ports.includes(31203), 420000, { afterId: after }); + expect(proven.data.reason).to.equal('proven'); + await waitForAppInstalled(env.clients[FIRST_LOST], 'udplossyapp', 300000, { afterId: after }); + }); + + // Two peers' silence is reported in health. Taking the node out of service for + // it waits on a policy document this fleet's policy does not carry. + it('reports UDP degraded in health, and stays in service, when the availability check confirms the failure', async function () { + this.timeout(600000); + await waitFor(async () => (await udpCheck(env.clients[BLOCKED])) === 'degraded', { + timeout: 540000, interval: 5000, label: 'the blocked node reports udp degraded', + }); + await waitFor(async () => (await udpCheck(env.clients[OPEN])) === 'ok', { + timeout: 60000, interval: 5000, label: 'the open node reports udp ok', + }); + const dos = (await env.clients[BLOCKED].getDOSState()).data; + expect(dos.dosMessage).to.equal(null); + }); +}); + +describe('a node whose peers do not probe UDP', function () { + const NODE = 0; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + stubPeers: [1, 2, 3], + networkShapes: { [NODE]: 'udp-blocked' }, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1, ...PORT_TEST } }, + }); + await bootAndPeer(env); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + // The first nodes to update have almost no peer that probes UDP. Silence they + // cannot get is not silence, so they install on TCP proof as before. + it('installs on TCP proof, its UDP unproven', async function () { + this.timeout(420000); + const after = env.clients[NODE].getLastEventId(); + const app = await appPinnedTo('udpoldpeersapp', 31204, NODE); + await seedSpawnerApp(env, app); + + const unproven = await env.clients[NODE].waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); + expect(unproven.data.reason).to.equal('udpUnproven'); + expect(unproven.data.peers).to.deep.equal([]); + await waitForAppInstalled(env.clients[NODE], 'udpoldpeersapp', 300000, { afterId: after }); + }); +}); From 372db11520659b743ebf56c8a89eb24d4b015987 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 08:49:10 +0100 Subject: [PATCH 153/369] test(harness): 3004 names ports below the range the harness hands out Co-Authored-By: Claude Opus 5.5 --- .../runner/tests/3004-a-port-is-proven-over-udp.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index 61ff346b0c..1d43f8ddf8 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -100,10 +100,10 @@ describe('a port is proven over UDP', function () { it('installs on a node whose port answers over TCP and UDP, proven by one peer', async function () { this.timeout(420000); const after = env.clients[OPEN].getLastEventId(); - const app = await appPinnedTo('udpopenapp', 31201, OPEN); + const app = await appPinnedTo('udpopenapp', 31141, OPEN); await seedSpawnerApp(env, app); - const proven = await env.clients[OPEN].waitForEvent('ports:proven', (d) => d.ports.includes(31201), 420000, { afterId: after }); + const proven = await env.clients[OPEN].waitForEvent('ports:proven', (d) => d.ports.includes(31141), 420000, { afterId: after }); expect(proven.data.reason).to.equal('proven'); await waitForAppInstalled(env.clients[OPEN], 'udpopenapp', 300000, { afterId: after }); }); @@ -111,10 +111,10 @@ describe('a port is proven over UDP', function () { it('refuses a port two peers hear nothing on over UDP', async function () { this.timeout(420000); const after = env.clients[BLOCKED].getLastEventId(); - const app = await appPinnedTo('udpblockedapp', 31202, BLOCKED); + const app = await appPinnedTo('udpblockedapp', 31142, BLOCKED); await seedSpawnerApp(env, app); - const refused = await env.clients[BLOCKED].waitForEvent('ports:udpUnreachable', (d) => d.port === 31202, 420000, { afterId: after }); + const refused = await env.clients[BLOCKED].waitForEvent('ports:udpUnreachable', (d) => d.port === 31142, 420000, { afterId: after }); expect(refused.data.reason).to.equal('udpUnreachable'); expect(refused.data.peers).to.have.length(2); expect(new Set(refused.data.peers).size).to.equal(2); @@ -131,10 +131,10 @@ describe('a port is proven over UDP', function () { it('proves a port whose first datagram from each peer is lost', async function () { this.timeout(420000); const after = env.clients[FIRST_LOST].getLastEventId(); - const app = await appPinnedTo('udplossyapp', 31203, FIRST_LOST); + const app = await appPinnedTo('udplossyapp', 31143, FIRST_LOST); await seedSpawnerApp(env, app); - const proven = await env.clients[FIRST_LOST].waitForEvent('ports:proven', (d) => d.ports.includes(31203), 420000, { afterId: after }); + const proven = await env.clients[FIRST_LOST].waitForEvent('ports:proven', (d) => d.ports.includes(31143), 420000, { afterId: after }); expect(proven.data.reason).to.equal('proven'); await waitForAppInstalled(env.clients[FIRST_LOST], 'udplossyapp', 300000, { afterId: after }); }); @@ -181,7 +181,7 @@ describe('a node whose peers do not probe UDP', function () { it('installs on TCP proof, its UDP unproven', async function () { this.timeout(420000); const after = env.clients[NODE].getLastEventId(); - const app = await appPinnedTo('udpoldpeersapp', 31204, NODE); + const app = await appPinnedTo('udpoldpeersapp', 31144, NODE); await seedSpawnerApp(env, app); const unproven = await env.clients[NODE].waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); From 9babc46728a2fdea11e3a7b8ee499fa1df90f8a2 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 09:04:56 +0100 Subject: [PATCH 154/369] test(harness): 3003 the availability check maps its port under test for TCP and UDP Co-Authored-By: Claude Opus 5.5 --- ...se-router-refuses-it-is-out-of-service-until-it-stops.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js index 6318c43dbc..4d3631589c 100644 --- a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js @@ -120,7 +120,7 @@ describe('a node whose router refuses it is out of service until it stops', func await env?.teardown(); }); - it('tests its ports over TCP alone, and waits the long wait after a test that passed', async function () { + it('tests its ports over TCP and UDP, and waits the long wait after a test that passed', async function () { this.timeout(300000); let state = { requests: [] }; await waitFor(async () => { @@ -131,7 +131,9 @@ describe('a node whose router refuses it is out of service until it stops', func }, { timeout: 240000, interval: 2000, label: 'a test mapping made and removed' }); const made = testMappings(state.requests); - expect(made.map((r) => r.protocol), 'TCP only').to.deep.equal(made.map(() => 'TCP')); + const [first] = made; + expect(made.filter((r) => r.port === first.port).map((r) => r.protocol).sort(), 'both protocols of the port under test') + .to.deep.equal(['TCP', 'UDP']); // a failed test is retried within seconds; a passed one only after the long wait const quietFrom = Date.now(); From 754cbc0f5dfebd3e821b89df5afa509ad0a89d0a Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 09:09:24 +0100 Subject: [PATCH 155/369] test(harness): the UDP shapes drop on the fleet device alone, leaving loopback DNS Co-Authored-By: Claude Opus 5.5 --- test-infra/network-shapes.sh | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test-infra/network-shapes.sh b/test-infra/network-shapes.sh index 82b8b98bb3..a4684936c8 100755 --- a/test-infra/network-shapes.sh +++ b/test-infra/network-shapes.sh @@ -50,13 +50,14 @@ # direct-nameserver the node's resolv.conf names the fleet's resolver itself, # as a host that lists a public resolver does, rather than # Docker's embedded server in front of it. -# udp-blocked no UDP datagram starting a flow reaches the node, as behind -# a router that forwards TCP alone; replies to the node's own -# datagrams (DNS among them) still arrive. -# udp-first-lost the first UDP datagram from each source starting a flow is -# dropped, and every later one within 30 s arrives: loss that -# a probe sending once never survives and one that sends -# again always does. +# udp-blocked no UDP datagram starting a flow reaches the node on its +# fleet device, as behind a router that forwards TCP alone; +# replies to the node's own datagrams still arrive, and +# loopback (Docker's embedded DNS) is untouched. +# udp-first-lost the first UDP datagram from each source starting a flow on +# the fleet device is dropped, and every later one within +# 30 s arrives: loss that a probe sending once never survives +# and one that sends again always does. set -euo pipefail shape="$1" @@ -164,11 +165,11 @@ case "$shape" in printf 'nameserver %s\n' "$FLUX_E2E_RESOLVER" > /etc/resolv.conf ;; udp-blocked) - iptables -A INPUT -p udp -m conntrack --ctstate NEW -j DROP + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -j DROP ;; udp-first-lost) - iptables -A INPUT -p udp -m conntrack --ctstate NEW -m recent --name udpseen --rcheck --seconds 30 -j ACCEPT - iptables -A INPUT -p udp -m conntrack --ctstate NEW -m recent --name udpseen --set -j DROP + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m recent --name udpseen --rcheck --seconds 30 -j ACCEPT + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m recent --name udpseen --set -j DROP ;; *) echo "network-shapes: unknown shape '$shape'" >&2 From 2bd474750bb390f49f5b67803bb3746d492988b7 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 09:14:19 +0100 Subject: [PATCH 156/369] test(ports): a UDP reply from another source port proves the port; 50 silent ports fit the requester's wait Co-Authored-By: Claude Opus 5.5 --- tests/unit/portTestEcho.test.js | 50 +++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tests/unit/portTestEcho.test.js b/tests/unit/portTestEcho.test.js index 0079db8cdc..aafef02bd3 100644 --- a/tests/unit/portTestEcho.test.js +++ b/tests/unit/portTestEcho.test.js @@ -1,6 +1,7 @@ const http = require('node:http'); const { EventEmitter } = require('node:events'); const { expect } = require('chai'); +const config = require('config'); const sinon = require('sinon'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); @@ -157,6 +158,27 @@ describe('the pre-install port test over UDP, end to end', () => { expect(seen).to.be.at.least(2); }); + // A NAT in front of the requester can rewrite the reply's source port, so + // the reply is taken from whatever port it comes from. + it('takes a reply that comes from another source port', async () => { + const probed = dgram.createSocket('udp4'); + const rewritten = dgram.createSocket('udp4'); + await new Promise((resolve) => { probed.bind(0, '127.0.0.1', resolve); }); + await new Promise((resolve) => { rewritten.bind(0, '127.0.0.1', resolve); }); + probed.on('message', (message, rinfo) => { + rewritten.send(Buffer.from(`FLUXPT ${TOKEN}`), rinfo.port, rinfo.address); + }); + + const probedPort = probed.address().port; + const replyPort = rewritten.address().port; + const answer = await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', probedPort, QUICK); + probed.close(); + rewritten.close(); + + expect(replyPort, 'the reply came from the port probed').to.not.equal(probedPort); + expect(answer).to.include(TOKEN); + }); + it('says nothing about a port it could not send to', async () => { expect(await fluxNetworkHelper.portAnsweredUdp('::1', port, QUICK)).to.equal(undefined); }); @@ -377,6 +399,34 @@ describe('checkAppAvailability probes the address that asked', () => { expect(answer.data.message).to.include(`Failed port: ${closedPort}`); }); + // The largest app's ports, every one of them silent over UDP: each costs a + // probe's full deadline, and all of them have to be answered for inside the + // time the requester waits for this peer. + it('answers for 50 ports silent over UDP inside the requester\'s wait', async function () { + this.timeout(60000); + const servers = await Promise.all(Array.from({ length: 50 }, async () => { + const s = new FluxHttpTestServer(TOKEN); + await new Promise((resolve) => s.listen(0, HERE, resolve)); + return s; + })); + const ports = servers.map((s) => s.address().port); + + const started = Date.now(); + const answer = await ask( + { + ip: HERE, port: 16127, ports, pubKey: 'k', signature: 's', echo: true, udp: true, + }, + HERE, + ); + const elapsed = Date.now() - started; + await Promise.all(servers.map((s) => new Promise((resolve) => s.close(resolve)))); + + expect(answer.status).to.equal('success'); + expect(Object.values(answer.data.udpAnswered)).to.have.length(50); + expect(Object.values(answer.data.udpAnswered).every((reply) => reply === null)).to.equal(true); + expect(elapsed, 'outlasted the requester\'s wait for a peer').to.be.below(config.fluxapps.portTestPeerTimeoutMs); + }); + // maxComponents (10) x ports per component (5) in appValidator. Each port // costs a connect timeout and they are tested in sequence. it('refuses more ports than an application could hold', async () => { From 94bd127d95f7be7da68bc09a7bd673db0a5a8a09 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 09:15:14 +0100 Subject: [PATCH 157/369] test(harness): 3004 proves UDP through ufw, under steady loss and behind UPnP, and counts it under policy The open node runs ufw, so its UDP test rule is what admits the probes, and no test rule is left once the app installs. udp-every-other drops every other datagram starting a flow. The every-other node is behind a UPnP gateway, which is asked for both protocols of the app's port. A fleet whose policy carries udpportdos takes a node closed over UDP out of service. Co-Authored-By: Claude Opus 5.5 --- test-infra/network-shapes.sh | 5 + .../tests/3004-a-port-is-proven-over-udp.js | 145 +++++++++++++----- 2 files changed, 111 insertions(+), 39 deletions(-) diff --git a/test-infra/network-shapes.sh b/test-infra/network-shapes.sh index a4684936c8..55f721b2a4 100755 --- a/test-infra/network-shapes.sh +++ b/test-infra/network-shapes.sh @@ -58,6 +58,8 @@ # the fleet device is dropped, and every later one within # 30 s arrives: loss that a probe sending once never survives # and one that sends again always does. +# udp-every-other every other UDP datagram starting a flow on the fleet +# device is dropped, whatever its source: steady loss of half. set -euo pipefail shape="$1" @@ -167,6 +169,9 @@ case "$shape" in udp-blocked) iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -j DROP ;; + udp-every-other) + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m statistic --mode nth --every 2 --packet 0 -j DROP + ;; udp-first-lost) iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m recent --name udpseen --rcheck --seconds 30 -j ACCEPT iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m recent --name udpseen --set -j DROP diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index 1d43f8ddf8..a266331e1d 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -8,7 +8,9 @@ import { getSubnetConfig, REGISTRY_REPO_HOST } from '../framework/subnet-config. import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; -import { waitFor, waitForAppInstalled } from '../framework/wait.js'; +import { execInContainer, restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; +import { waitFor, waitForAppInstalled, waitForBootSettled } from '../framework/wait.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // Every app port is published for TCP and UDP, so a node proves both before it @@ -17,17 +19,23 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // UDP has no handshake: silence is all a closed port and a lost datagram say. A // peer probes each port and hands back what replied; only the node's own token // coming back proves a port, and a refusal needs two distinct peers that heard -// nothing. The shapes put the fault in the node's own namespace: +// nothing. The shapes put the fault in the node's own namespace, on its fleet +// device: // -// udp-blocked no datagram starting a flow arrives - a router forwarding TCP -// alone. Refused, and the availability check reports it. -// udp-first-lost the first datagram from each source is lost, every later one -// arrives. A probe that sends once never gets through; one that -// sends again always does. Proven. +// udp-blocked no datagram starting a flow arrives - a router forwarding TCP +// alone. Refused, and the availability check reports it. +// udp-first-lost the first datagram from each source is lost, every later one +// arrives. A probe that sends once never gets through; one that +// sends again always does. Proven. +// udp-every-other every other datagram is lost, whatever its source. Proven. // -// The peers in the first fleet are real nodes on this branch. The second fleet's -// peers are stubs that read TCP and do not probe UDP - the state of the network -// while it updates - and a node there must install on TCP proof alone. +// The open node runs ufw as an install leaves it, so its UDP test rule is what +// admits the probes, and it is gone once the test ends. The every-other node is +// behind a UPnP gateway, which must be asked for both protocols of the port. +// +// The peers in the first two fleets are real nodes on this branch. The last +// fleet's peers are stubs that read TCP and do not probe UDP - the state of the +// network while it updates - and a node there must install on TCP proof alone. const subnet = getSubnetConfig(); const socketAddressOf = (index) => `${subnet.nodeIp(index + 1)}:16127`; @@ -44,6 +52,8 @@ const APPS_PORT_CHECK = { // Enough attempts that a node always reaches two peers that answer. const PORT_TEST = { portTestMaxAttempts: 3 }; +const PORTS_DOS_PREFIX = 'Ports tested not reachable from outside'; + const appPinnedTo = async (name, port, index) => { await pushImage(name, 'v1'); return buildSeedableApp({ @@ -70,25 +80,45 @@ const appPinnedTo = async (name, port, index) => { const udpCheck = async (client) => (await client.get('/flux/health').catch(() => null))?.data?.udp; +// Seeds an app pinned to one node and waits for that node's port test to prove +// its port, then for the install. +const provenAndInstalled = async (env, index, name, port) => { + const client = env.clients[index]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo(name, port, index)); + const proven = await client.waitForEvent('ports:proven', (d) => d.ports.includes(port), 420000, { afterId: after }); + expect(proven.data.reason).to.equal('proven'); + await waitForAppInstalled(client, name, 300000, { afterId: after }); +}; + describe('a port is proven over UDP', function () { const OPEN = 0; const BLOCKED = 1; const FIRST_LOST = 2; + const EVERY_OTHER = 3; let env; dumpLogsOnFailure(() => env); before(async function () { - this.timeout(600000); + this.timeout(900000); env = await createTestEnv({ hookCtx: this, nodes: 4, - networkShapes: { [BLOCKED]: 'udp-blocked', [FIRST_LOST]: 'udp-first-lost' }, + firewall: [OPEN], + networkShapes: { [BLOCKED]: 'udp-blocked', [FIRST_LOST]: 'udp-first-lost', [EVERY_OTHER]: 'udp-every-other' }, configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, nodeConfigOverrides: { [OPEN]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } }, [BLOCKED]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } }, }, }); + // the first boot writes userconfig.js (its identity); a node with an api port + // in it verifies UPnP when it starts + const upnp = env.clients[EVERY_OTHER]; + await waitForBootSettled(upnp); + await enableUpnpGateway(subnet.nodeIp(EVERY_OTHER + 1)); + await setNodeApiPort(upnp.container, 16127); + await restartFluxos(upnp.container); await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); }); @@ -97,46 +127,48 @@ describe('a port is proven over UDP', function () { await env?.teardown(); }); - it('installs on a node whose port answers over TCP and UDP, proven by one peer', async function () { - this.timeout(420000); - const after = env.clients[OPEN].getLastEventId(); - const app = await appPinnedTo('udpopenapp', 31141, OPEN); - await seedSpawnerApp(env, app); + it('installs on a firewalled node whose port answers over TCP and UDP, and leaves no test rule behind', async function () { + this.timeout(720000); + await provenAndInstalled(env, OPEN, 'udpopenapp', 31141); - const proven = await env.clients[OPEN].waitForEvent('ports:proven', (d) => d.ports.includes(31141), 420000, { afterId: after }); - expect(proven.data.reason).to.equal('proven'); - await waitForAppInstalled(env.clients[OPEN], 'udpopenapp', 300000, { afterId: after }); + const rules = (await execInContainer(env.clients[OPEN].container, 'ufw status; true')).stdout + .split('\n').map((line) => line.trim()).filter((line) => line.startsWith('31141')); + expect(rules.some((line) => /^31141\s/.test(line)), 'canary: the app\'s own rule for its port').to.equal(true); + expect(rules.filter((line) => line.includes('/udp') || line.includes('/tcp')), 'a test rule left behind').to.deep.equal([]); }); it('refuses a port two peers hear nothing on over UDP', async function () { this.timeout(420000); - const after = env.clients[BLOCKED].getLastEventId(); - const app = await appPinnedTo('udpblockedapp', 31142, BLOCKED); - await seedSpawnerApp(env, app); + const client = env.clients[BLOCKED]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo('udpblockedapp', 31142, BLOCKED)); - const refused = await env.clients[BLOCKED].waitForEvent('ports:udpUnreachable', (d) => d.port === 31142, 420000, { afterId: after }); + const refused = await client.waitForEvent('ports:udpUnreachable', (d) => d.port === 31142, 420000, { afterId: after }); expect(refused.data.reason).to.equal('udpUnreachable'); expect(refused.data.peers).to.have.length(2); expect(new Set(refused.data.peers).size).to.equal(2); - await env.clients[BLOCKED].waitForEvent( + await client.waitForEvent( 'spawner:deferred', (d) => d.appName === 'udpblockedapp' && d.reason === 'ports_not_available', 60000, { afterId: after }, ); - const installed = await env.clients[BLOCKED].getInstalledApps(); + const installed = await client.getInstalledApps(); expect(installed.data.map((a) => a.name)).to.not.include('udpblockedapp'); }); it('proves a port whose first datagram from each peer is lost', async function () { - this.timeout(420000); - const after = env.clients[FIRST_LOST].getLastEventId(); - const app = await appPinnedTo('udplossyapp', 31143, FIRST_LOST); - await seedSpawnerApp(env, app); + this.timeout(720000); + await provenAndInstalled(env, FIRST_LOST, 'udplossyapp', 31143); + }); + + it('proves a port that loses every other datagram, behind UPnP mapped for both protocols', async function () { + this.timeout(720000); + await provenAndInstalled(env, EVERY_OTHER, 'udpsteadylossapp', 31145); - const proven = await env.clients[FIRST_LOST].waitForEvent('ports:proven', (d) => d.ports.includes(31143), 420000, { afterId: after }); - expect(proven.data.reason).to.equal('proven'); - await waitForAppInstalled(env.clients[FIRST_LOST], 'udplossyapp', 300000, { afterId: after }); + const { requests } = await upnpGatewayState(subnet.nodeIp(EVERY_OTHER + 1)); + const mapped = requests.filter((r) => r.action === 'AddPortMapping' && r.port === 31145 && r.description === 'Flux_App_udpsteadylossapp'); + expect([...new Set(mapped.map((r) => r.protocol))].sort()).to.deep.equal(['TCP', 'UDP']); }); // Two peers' silence is reported in health. Taking the node out of service for @@ -147,13 +179,48 @@ describe('a port is proven over UDP', function () { timeout: 540000, interval: 5000, label: 'the blocked node reports udp degraded', }); await waitFor(async () => (await udpCheck(env.clients[OPEN])) === 'ok', { - timeout: 60000, interval: 5000, label: 'the open node reports udp ok', + timeout: 60000, interval: 5000, label: 'the firewalled open node reports udp ok', }); const dos = (await env.clients[BLOCKED].getDOSState()).data; expect(dos.dosMessage).to.equal(null); }); }); +describe('a port closed over UDP, with the policy that counts it', function () { + const BLOCKED = 0; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: { [BLOCKED]: 'udp-blocked' }, + policy: { documents: { udpportdos: true } }, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + // the first port failure is walked, the next counts, and one count is the threshold + nodeConfigOverrides: { + [BLOCKED]: { fluxapps: { appsPortCheck: { ...APPS_PORT_CHECK, thresholds: { dos: 4, portsHighEdge: 1, portsLowEdge: 0 } } } }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('takes the node out of service once the failures it confirms reach the threshold', async function () { + this.timeout(600000); + await waitFor(async () => (await env.clients[BLOCKED].getDOSState()).data?.dosMessage?.startsWith(PORTS_DOS_PREFIX) === true, { + timeout: 540000, interval: 5000, label: 'the blocked node is out of service for its ports', + }); + expect(await udpCheck(env.clients[BLOCKED])).to.equal('degraded'); + }); +}); + describe('a node whose peers do not probe UDP', function () { const NODE = 0; let env; @@ -180,13 +247,13 @@ describe('a node whose peers do not probe UDP', function () { // cannot get is not silence, so they install on TCP proof as before. it('installs on TCP proof, its UDP unproven', async function () { this.timeout(420000); - const after = env.clients[NODE].getLastEventId(); - const app = await appPinnedTo('udpoldpeersapp', 31144, NODE); - await seedSpawnerApp(env, app); + const client = env.clients[NODE]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo('udpoldpeersapp', 31144, NODE)); - const unproven = await env.clients[NODE].waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); + const unproven = await client.waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); expect(unproven.data.reason).to.equal('udpUnproven'); expect(unproven.data.peers).to.deep.equal([]); - await waitForAppInstalled(env.clients[NODE], 'udpoldpeersapp', 300000, { afterId: after }); + await waitForAppInstalled(client, 'udpoldpeersapp', 300000, { afterId: after }); }); }); From acae4d20acd072794b588408776604876ec7ac88 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 09:36:48 +0100 Subject: [PATCH 158/369] test(harness): the policy stub publishes udpportdos, a refused policy fails the fleet, and the peer wait outlasts a UDP probe The stub's bundle carries udpportdos when a suite sets it. createTestEnv throws when the stub refuses the policy it was given, instead of booting the fleet on the default bundle. portTestPeerTimeoutMs is 8 s: a peer answers only once its probe of a silent UDP port reaches its 3 s deadline. Co-Authored-By: Claude Opus 5.5 --- test-infra/config/HARNESS_TUNABLES.md | 2 +- test-infra/config/shared.js | 5 ++++- test-infra/external-http-stub/index.js | 6 ++++++ test-infra/runner/framework/test-env.js | 5 ++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/test-infra/config/HARNESS_TUNABLES.md b/test-infra/config/HARNESS_TUNABLES.md index b3a116a520..6019ecaa1e 100644 --- a/test-infra/config/HARNESS_TUNABLES.md +++ b/test-infra/config/HARNESS_TUNABLES.md @@ -128,7 +128,7 @@ loses the property, so the factor is pinned by a unit test. | `fluxapps.portRestoreIntervalMs` | 600000 | 30000 | 20.0x | | `fluxapps.portTestBindDelayMs` | 5000 | 100 | 50.0x | | `fluxapps.portTestMaxAttempts` | 5 | 2 | 2.5x | -| `fluxapps.portTestPeerTimeoutMs` | 30000 | 3000 | 10.0x | +| `fluxapps.portTestPeerTimeoutMs` | 30000 | 8000 | 3.8x | | `fluxapps.portTestPropagationDelayMs` | 10000 | 100 | 100.0x | | `fluxapps.redeploy.composedDelay` | 5 | 1 | 5.0x | | `fluxapps.removal.delay` | 300 | 5 | 60.0x | diff --git a/test-infra/config/shared.js b/test-infra/config/shared.js index 8894091630..c66064f785 100644 --- a/test-infra/config/shared.js +++ b/test-infra/config/shared.js @@ -225,7 +225,10 @@ module.exports = { installCollisionWaitMs: 5000, portTestBindDelayMs: 100, portTestPropagationDelayMs: 100, - portTestPeerTimeoutMs: 3000, + // A peer answers only once its UDP probe of a silent port has reached its deadline + // (UDP_PROBE_DEADLINE_MS, 3 s, fluxNetworkHelper.js), so the wait for that peer has to + // outlast it, or every UDP refusal reads as a peer that never answered. + portTestPeerTimeoutMs: 8000, portTestMaxAttempts: 2, spawnReconfirmDelayMs: 30000, globalCmdDelayMs: 100, diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 3610c45830..cfd307968a 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -408,6 +408,9 @@ const state = { // enterprise node, which is what every suite that is not about enterprise placement wants. // Absent would be a different thing entirely - see the policy block below. enterpriseNodes: {}, + // Whether a port closed over UDP counts toward the availability check's DOS. null is + // the document absent from the published bundle, which is the default. + udpPortDos: null, latestRelease: { tag_name: 'v0.0.0', name: 'stub-release' }, geolocation: {}, // The syncthing the node image ships, read from the repository the image was built @@ -543,6 +546,7 @@ function resignPolicy({ bumpSeq = true } = {}) { vettedrepositories: state.vettedRepositories, tamperingblockednodes: state.tamperingBlocklist, enterprisenodes: state.enterpriseNodes, + ...(state.udpPortDos === null ? {} : { udpportdos: state.udpPortDos }), }, // Content-addressed, as the real publisher does it: the plain name is mutable and means // "the current table", so the bundle names the hash a fetch should produce. Derived from @@ -1267,6 +1271,7 @@ control.post('/policy', (req, res) => { vettedrepositories: 'vettedRepositories', tamperingblockednodes: 'tamperingBlocklist', enterprisenodes: 'enterpriseNodes', + udpportdos: 'udpPortDos', }; for (const [name, value] of Object.entries(body.documents)) { if (!map[name]) return res.status(400).json({ error: `unknown document ${name}` }); @@ -1314,6 +1319,7 @@ control.post('/reset', (req, res) => { state.vettedRepositories = []; state.tamperingBlocklist = []; state.enterpriseNodes = {}; + state.udpPortDos = null; state.latestRelease = { tag_name: 'v0.0.0', name: 'stub-release' }; state.geolocation = {}; artifacts.clear(); diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index f3b885adb8..e09fd5b769 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -1523,11 +1523,14 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // Takes the stub's own /policy control body: { documents }, { signer: 'rogue' }, // { seq }, { available: false } to boot a fleet whose source never answers. if (policy) { - await fetch(`http://${EXTERNAL_STUB_IP}:3001/policy`, { + const published = await fetch(`http://${EXTERNAL_STUB_IP}:3001/policy`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(policy), }); + if (!published.ok) { + throw new Error(`createTestEnv: the stub refused the policy (${published.status}): ${await published.text()}`); + } } // Names the fleet's resolver answers itself, set before any node starts so the first From 8fe628072e7215c6fd80168b9a65e58a6c5e09d8 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 09:42:04 +0100 Subject: [PATCH 159/369] test(harness): 3004 reads the udp check from an unfit node's health under data.checks Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/tests/3004-a-port-is-proven-over-udp.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index a266331e1d..1aafea1863 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -78,7 +78,11 @@ const appPinnedTo = async (name, port, index) => { }); }; -const udpCheck = async (client) => (await client.get('/flux/health').catch(() => null))?.data?.udp; +// A fit node answers with its checks as the data; an unfit one, under data.checks. +const udpCheck = async (client) => { + const health = await client.get('/flux/health').catch(() => null); + return health?.status === 'success' ? health.data?.udp : health?.data?.checks?.udp; +}; // Seeds an app pinned to one node and waits for that node's port test to prove // its port, then for the install. From 0209fdacf9c5b145c51077dad576c1f34d4c8137 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 11:32:08 +0100 Subject: [PATCH 160/369] fix(ports): two silent peers refuse a port over UDP only once no attempt left proves it A peer that cannot hear UDP replies is silent for every node it probes, so silence is weighed after the loop: every attempt left is spent looking for a peer that hears our token, and two silent peers refuse the install only when none did. 3004 boots a node whose other peers include two that never hear a UDP reply (udp-replies-lost); the node's port is proven by the third, whichever order the peers are asked in. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appNetwork/portManager.js | 42 ++++++++++--------- test-infra/network-shapes.sh | 6 +++ .../tests/3004-a-port-is-proven-over-udp.js | 32 ++++++++++++++ tests/unit/portManager.test.js | 11 ++++- 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 03b7f4aa06..c517cf4234 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -1158,25 +1158,11 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { continue; } - if (udpClosedPorts.length) { - udpDisagreements.set(askingIP, udpClosedPorts); - const disputed = udpDisputes(udpDisagreements, udpProven); - const udpRefused = refusedPort(disputed, PORT_TEST_CORROBORATION); - if (udpRefused !== null) { - const readings = [...disputed.entries()].map(([peer, port]) => `${peer} on port ${port}`); - log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} at this address does not answer this node over UDP, as heard by ` - + `${disputed.size} peers (${readings.join(', ')}). Installation aborted.`); - decision = { - ok: false, - reason: 'udpUnreachable', - port: udpRefused, - peers: [...disputed.keys()], - readings: Object.fromEntries(disputed), - }; - fluxEventBus.publish('ports:udpUnreachable', decision); - break; - } - } + // UDP silence is decided after the loop: a peer that cannot hear UDP + // replies is silent for every node it probes, so every attempt left is + // spent looking for a peer that proves the port before two silent ones + // refuse it. + if (udpClosedPorts.length) udpDisagreements.set(askingIP, udpClosedPorts); if (reportedPort === null) { // eslint-disable-next-line no-continue @@ -1211,6 +1197,24 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { fluxEventBus.publish('ports:notOurs', decision); } + if (decision === null) { + const disputed = udpDisputes(udpDisagreements, udpProven); + const udpRefused = refusedPort(disputed, PORT_TEST_CORROBORATION); + if (udpRefused !== null) { + const readings = [...disputed.entries()].map(([peer, port]) => `${peer} on port ${port}`); + log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} at this address does not answer this node over UDP, as heard by ` + + `${disputed.size} peers (${readings.join(', ')}) and proven by none of the ${asked.length} asked. Installation aborted.`); + decision = { + ok: false, + reason: 'udpUnreachable', + port: udpRefused, + peers: [...disputed.keys()], + readings: Object.fromEntries(disputed), + }; + fluxEventBus.publish('ports:udpUnreachable', decision); + } + } + if (decision === null) { // Nothing proved and nothing corroborated. What was not learned does not // refuse an install: this is the check this node made before the token diff --git a/test-infra/network-shapes.sh b/test-infra/network-shapes.sh index 55f721b2a4..9babc7cb9b 100755 --- a/test-infra/network-shapes.sh +++ b/test-infra/network-shapes.sh @@ -60,6 +60,9 @@ # and one that sends again always does. # udp-every-other every other UDP datagram starting a flow on the fleet # device is dropped, whatever its source: steady loss of half. +# udp-replies-lost no reply to a UDP datagram the node sent arrives on its fleet +# device: a peer that probes a port and never hears back, +# whatever the port does. set -euo pipefail shape="$1" @@ -169,6 +172,9 @@ case "$shape" in udp-blocked) iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -j DROP ;; + udp-replies-lost) + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate ESTABLISHED -j DROP + ;; udp-every-other) iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m statistic --mode nth --every 2 --packet 0 -j DROP ;; diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index 1aafea1863..ca34312402 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -28,6 +28,9 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // arrives. A probe that sends once never gets through; one that // sends again always does. Proven. // udp-every-other every other datagram is lost, whatever its source. Proven. +// udp-replies-lost on a PEER: it never hears a reply to its own probes, so it is +// silent about every node. Two such peers do not refuse a node +// while a peer that proves its port is still to ask. // // The open node runs ufw as an install leaves it, so its UDP test rule is what // admits the probes, and it is gone once the test ends. The every-other node is @@ -190,6 +193,35 @@ describe('a port is proven over UDP', function () { }); }); +describe('a node whose first peers cannot hear UDP replies', function () { + const NODE = 0; + const DEAF = [1, 2]; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: Object.fromEntries(DEAF.map((i) => [i, 'udp-replies-lost'])), + // every other node is asked, the two deaf ones among them + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, portTestMaxAttempts: 3 } }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('is proven by the peer that hears it, whichever order the peers are asked in', async function () { + this.timeout(720000); + await provenAndInstalled(env, NODE, 'udpdeafpeersapp', 31146); + }); +}); + describe('a port closed over UDP, with the policy that counts it', function () { const BLOCKED = 0; let env; diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index 5e5e4613f2..ef3a99396c 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -1442,7 +1442,7 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(udpServers[0].allow.secondCall.calledBefore(axios.post.secondCall)).to.equal(true); }); - it('refuses once two distinct peers hear nothing over UDP', async () => { + it('refuses once two distinct peers hear nothing over UDP and no peer proves it', async () => { const result = await withPeers([answering(null), answering(null), UNREACHABLE, UNREACHABLE, UNREACHABLE]); expect(result.ok, 'two peers hearing nothing did not refuse').to.equal(false); @@ -1458,6 +1458,15 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(result.reason).to.equal('udpUnreachable'); }); + // A peer that cannot hear UDP replies is silent for every node. Two of them + // first do not refuse while a peer that can prove the port is still to ask. + it('asks every attempt left for proof after two peers hear nothing', async () => { + const result = await withPeers([answering(null), answering(null), answering('ours')]); + + expect(result).to.deep.equal({ ok: true, reason: 'proven', port: null }); + sinon.assert.callCount(axios.post, 3); + }); + // UDP loses datagrams. Our token heard by one peer is proof, and another // peer's silence on the same port is loss. it('takes our token heard by one peer over silence heard by another', async () => { From 1dbbbfe72560a4db3877d268688120bf5fc2d89a Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 11:35:51 +0100 Subject: [PATCH 161/369] test(harness): 3004's UDP refusal names a silent peer for every attempt Co-Authored-By: Claude Opus 5.5 --- .../runner/tests/3004-a-port-is-proven-over-udp.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index ca34312402..348e757419 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -150,10 +150,13 @@ describe('a port is proven over UDP', function () { const after = client.getLastEventId(); await seedSpawnerApp(env, await appPinnedTo('udpblockedapp', 31142, BLOCKED)); + // Every peer is silent here, and the node may ask as many as it has + // attempts. Silence is weighed only once no attempt is left that could prove + // the port, so the refusal names a peer for every attempt. const refused = await client.waitForEvent('ports:udpUnreachable', (d) => d.port === 31142, 420000, { afterId: after }); expect(refused.data.reason).to.equal('udpUnreachable'); - expect(refused.data.peers).to.have.length(2); - expect(new Set(refused.data.peers).size).to.equal(2); + expect(refused.data.peers, 'refused before every attempt was spent').to.have.length(PORT_TEST.portTestMaxAttempts); + expect(new Set(refused.data.peers).size).to.equal(PORT_TEST.portTestMaxAttempts); await client.waitForEvent( 'spawner:deferred', (d) => d.appName === 'udpblockedapp' && d.reason === 'ports_not_available', @@ -216,7 +219,7 @@ describe('a node whose first peers cannot hear UDP replies', function () { await env?.teardown(); }); - it('is proven by the peer that hears it, whichever order the peers are asked in', async function () { + it('is proven by the peer that hears it', async function () { this.timeout(720000); await provenAndInstalled(env, NODE, 'udpdeafpeersapp', 31146); }); From 798532780c7267a80e8941f34ae80d99178dc896 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 11:58:11 +0100 Subject: [PATCH 162/369] feat(ports): a port test names its protocols; behind UPnP the availability check tests one a pass /flux/checkappavailability takes `protocols`, tcp, udp or both, in place of `udp: true`. TCP alone when a caller names none; with both, UDP is probed once TCP has passed; with udp alone the port is never read over TCP. The pre-install test asks for both. A node behind UPnP maps one protocol per availability pass, TCP and UDP in turn, so a pass costs its router one mapping and one removal. It turns to the other protocol after a pass that passed, and stays on a protocol whose test is failing. A refusal of either protocol's mapping is a router fault, counted as one. On a UDP pass the peer's TCP answer is no evidence: a peer that does not know `protocols` tests TCP and finds it unmapped. A UDP pass whose peer does not probe UDP is tested again with another peer, up to three passes, before the turn goes back to TCP. A node with no mapping to make tests both protocols every pass. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/availabilityChecker.js | 119 ++++++++++++------ .../src/services/appNetwork/portManager.js | 5 +- ZelBack/src/services/fluxNetworkHelper.js | 16 ++- tests/unit/availabilityChecker.test.js | 82 +++++++++++- tests/unit/portManager.test.js | 2 +- tests/unit/portTestEcho.test.js | 37 +++++- 6 files changed, 209 insertions(+), 52 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index 02ddf9add0..e71aa4cd3a 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -23,6 +23,7 @@ async function handleTestShutdown(testingPort, testHttpServer, isArcane, options const skipFirewall = options.skipFirewall || false; const skipUpnp = options.skipUpnp || false; const skipHttpServer = options.skipHttpServer || false; + const protocols = options.protocols || ['tcp', 'udp']; if (options.udpTestServer) options.udpTestServer.close(); @@ -32,12 +33,13 @@ async function handleTestShutdown(testingPort, testHttpServer, isArcane, options || await fluxNetworkHelper.isFirewallActive().catch(() => true); if (updateFirewall) { - await fluxNetworkHelper - .deleteTestPortRule(testingPort, 'tcp') - .catch((e) => log.error(e)); - await fluxNetworkHelper - .deleteTestPortRule(testingPort, 'udp') - .catch((e) => log.error(e)); + // eslint-disable-next-line no-restricted-syntax + for (const protocol of protocols) { + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper + .deleteTestPortRule(testingPort, protocol) + .catch((e) => log.error(e)); + } } if (!skipUpnp) { @@ -123,12 +125,24 @@ function udpDosEnabled() { } /** - * How many passes a port one peer heard nothing on over UDP is tested again, - * looking for a second peer that probes UDP, before that peer's silence is - * dropped unconfirmed. + * How many passes a port is tested again over UDP, looking for a peer that + * probes UDP - to confirm one peer's silence, or on a UDP pass that learned + * nothing - before the pass is left unsettled. */ const UDP_RETESTS = 3; +/** + * The protocols a pass tests. A node behind UPnP maps one protocol per pass, + * TCP and UDP in turn, so a pass costs its router one mapping and one removal; + * a node with no mapping to make tests both every pass. + * @param {boolean} isUpnp + * @param {object} dosState + * @returns {string[]} + */ +function passProtocols(isUpnp, dosState) { + return isUpnp ? [dosState.testingProtocol ?? 'tcp'] : ['tcp', 'udp']; +} + /** * Whether an application the network lists at this node's public address holds * a port. Another node behind the same address serving that port is not a fault @@ -356,8 +370,12 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin return; } + const protocols = passProtocols(isUpnp, dosState); + const tcpTested = protocols.includes('tcp'); + const udpTested = protocols.includes('udp'); + // A port one peer heard nothing on over UDP is tested again by another. - const udpSuspect = dosState.udpSuspect?.port === dosState.testingPort ? dosState.udpSuspect : null; + const udpSuspect = udpTested && dosState.udpSuspect?.port === dosState.testingPort ? dosState.udpSuspect : null; // An external observer: this asks a peer whether it can reach US, and a Flux // node sharing our public address cannot answer that. Null when there is no @@ -380,12 +398,17 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const firewallActive = isArcane ? true : await fluxNetworkHelper.isFirewallActive(); if (firewallActive) { - await fluxNetworkHelper.allowTestPort(dosState.testingPort, 'tcp'); - await fluxNetworkHelper.allowTestPort(dosState.testingPort, 'udp'); + // eslint-disable-next-line no-restricted-syntax + for (const protocol of protocols) { + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.allowTestPort(dosState.testingPort, protocol); + } } if (isUpnp) { - const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App'); + // A router refusing either protocol's mapping for a reason of its own is + // a router fault, counted as one. + const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App', { protocols: protocols.map((protocol) => protocol.toUpperCase()) }); routerRefused = routerRefusedSinceLastPass(dosState); if (routerRefused) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); if (!upnpMapResult.mapped) { @@ -394,6 +417,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipHttpServer: true, + protocols, }); const upnpDelay = dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error; await waitForNextPass(upnpDelay); @@ -402,22 +426,25 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } } - const listening = new Promise((resolve, reject) => { - testHttpServer - .once('error', (err) => { - testHttpServer.removeAllListeners('listening'); - reject(err.message); - }) - .once('listening', () => { - testHttpServer.removeAllListeners('error'); - resolve(null); - }); - testHttpServer.listen(dosState.testingPort); - }); - - let error = await listening.catch((err) => err); - const httpListening = !error; - if (httpListening) { + let error = null; + let httpListening = false; + if (tcpTested) { + const listening = new Promise((resolve, reject) => { + testHttpServer + .once('error', (err) => { + testHttpServer.removeAllListeners('listening'); + reject(err.message); + }) + .once('listening', () => { + testHttpServer.removeAllListeners('error'); + resolve(null); + }); + testHttpServer.listen(dosState.testingPort); + }); + error = await listening.catch((err) => err); + httpListening = !error; + } + if (!error && udpTested) { udpTestServer = new fluxUdpTestServer.FluxUdpTestServer(portTestToken); error = await udpTestServer.listen(dosState.testingPort).then(() => null, (err) => err.message); } @@ -429,6 +456,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin skipUpnp: !isUpnp, skipHttpServer: !httpListening, udpTestServer, + protocols, }); await waitForNextPass(timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); @@ -455,16 +483,16 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin appname: 'appPortsTest', ports: [dosState.testingPort], pubKey: signer.pubKey, - // Read the port back rather than merely reach it, and probe it over UDP. + // Read the port back rather than merely reach it, over this pass's protocols. echo: true, - udp: true, + protocols, }; const signature = signer.sign(JSON.stringify(data)); if (!signature) throw new Error('checkMyAppsAvailability - the port test could not be signed'); data.signature = signature; - udpTestServer.allow(remoteIp); + if (udpTestServer) udpTestServer.allow(remoteIp); const resMyAppAvailability = await axios .post(`http://${remoteIp}:${remotePort}/flux/checkappavailability`, JSON.stringify(data), axiosConfig) @@ -479,7 +507,9 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipUpnp: !isUpnp, + skipHttpServer: !httpListening, udpTestServer, + protocols, }); if (!resMyAppAvailability) { @@ -503,12 +533,14 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin } const { answered, udpAnswered } = resMyAppAvailability.data.data || {}; - let portTestFailed = responseStatus === 'error'; + // A pass that does not test TCP reads nothing from the peer's TCP answer: a + // peer that does not know `protocols` tests TCP anyway, and finds it unmapped. + let portTestFailed = tcpTested && responseStatus === 'error'; // A peer that read the port: only our token says this node is what // answered. Anything else holding the port leaves it as unusable here as // nothing answering. A peer that does not read ports is taken at its word. - if (!portTestFailed && answered && portManager.portNotOurs([dosState.testingPort], answered, portTestToken) !== null) { + if (tcpTested && !portTestFailed && answered && portManager.portNotOurs([dosState.testingPort], answered, portTestToken) !== null) { log.warn(`checkMyAppsAvailability - Port ${dosState.testingPort} answered ${remoteIp}:${remotePort} as something other than this node`); portTestFailed = true; } @@ -516,7 +548,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // Over UDP, our token heard by one peer proves the port; silence heard by // one peer is tested again by another, and counts once two have heard it. let retestUdp = false; - const udp = portTestFailed ? 'unknown' : udpVerdict(udpAnswered, dosState.testingPort, portTestToken); + const udp = !udpTested || portTestFailed ? 'unknown' : udpVerdict(udpAnswered, dosState.testingPort, portTestToken); if (udp === 'proven') { // eslint-disable-next-line no-param-reassign dosState.udpSuspect = null; @@ -543,9 +575,19 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin retestUdp = udpSuspect.retests < UDP_RETESTS; // eslint-disable-next-line no-param-reassign if (!retestUdp) dosState.udpSuspect = null; + } else if (udpTested && !tcpTested) { + // A UDP pass whose peer does not probe UDP learned nothing. + // eslint-disable-next-line no-param-reassign + dosState.udpUnsettled = (dosState.udpUnsettled ?? 0) + 1; + retestUdp = dosState.udpUnsettled < UDP_RETESTS; + } + if (udp !== 'unknown' || !retestUdp) { + // eslint-disable-next-line no-param-reassign + dosState.udpUnsettled = 0; } let waitMs = 0; + let passed = false; if (portTestFailed && portsNotWorking.size < thresholds.portsHighEdge) { portsNotWorking.add(dosState.testingPort); @@ -576,6 +618,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin setRandomPort(); waitMs = timeouts.failure; } else { + passed = true; portsNotWorking.clear(); // eslint-disable-next-line no-param-reassign dosState.nextTestingPort = null; @@ -600,6 +643,10 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // eslint-disable-next-line no-param-reassign dosState.nextTestingPort = dosState.testingPort; waitMs = Math.min(waitMs, timeouts.failure); + } else if (isUpnp && passed) { + // The other protocol's turn, once this one has passed. + // eslint-disable-next-line no-param-reassign + dosState.testingProtocol = tcpTested ? 'udp' : 'tcp'; } // A router that refused this node a mapping or has yet to remove one is @@ -614,7 +661,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { - await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp, udpTestServer }); + await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp, udpTestServer, protocols: passProtocols(isUpnp, dosState) }); log.error(`checkMyAppsAvailability - Error: ${error}`); await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index c517cf4234..2b534c5df5 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -1000,8 +1000,9 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { // it should find is deliberately not here: a peer that never learns it // cannot produce it without actually reaching us. echo: true, - // And to probe each port over UDP, handing back what replied. - udp: true, + // Over both protocols: UDP is probed once TCP has passed, and what replied + // is handed back. + protocols: ['tcp', 'udp'], }; const stringData = JSON.stringify(data); // eslint-disable-next-line no-await-in-loop diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 8b40b1959e..a7d4d122b6 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -756,8 +756,14 @@ async function checkAppAvailability(req, res) { // null, rather than the test ending at the first port that failed. Only // with echo: the answers are what it returns. const everyPort = echo && processedBody.everyPort === true; - // Each port probed over UDP as well, once TCP has passed. - const udp = processedBody.udp === true; + // The protocols to test each port over. TCP alone when the caller names + // none; with both, UDP is probed once TCP has passed. + const protocols = processedBody.protocols ?? ['tcp']; + if (!Array.isArray(protocols) || !protocols.length || !protocols.every((protocol) => protocol === 'tcp' || protocol === 'udp')) { + throw new Error('Protocols to test must be tcp, udp or both'); + } + const tcp = protocols.includes('tcp'); + const udp = protocols.includes('udp'); const answered = {}; const testable = ports.filter((port) => { @@ -770,11 +776,11 @@ async function checkAppAvailability(req, res) { return withinRange && !iBP; }); - if (everyPort) { + if (tcp && everyPort) { await forEachLimited(testable, PORT_PROBE_CONCURRENCY, async (port) => { answered[port] = await portAnswered(ip, port); }); - } else { + } else if (tcp) { // eslint-disable-next-line no-restricted-syntax for (const port of testable) { if (echo) { @@ -810,7 +816,7 @@ async function checkAppAvailability(req, res) { // available from this peer, which is not the same as the ports being bad. // Added as a field rather than through createSuccessMessage, whose second // and third parameters are name and code. - if (echo) successResponse.data.answered = answered; + if (echo && tcp) successResponse.data.answered = answered; // Present at all says this peer probed UDP, as `answered` does for TCP. if (udp) successResponse.data.udpAnswered = udpAnswered; res.json(successResponse); diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 6d9138896a..f2551ef875 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -426,10 +426,82 @@ describe('availabilityChecker tests', () => { expect(mockDosState.portFailureScore ?? 0).to.equal(0); }); - it('maps the port under test over TCP and UDP', async () => { + // A pass costs the router one mapping and one removal. + it('maps the port under test for one protocol a pass, TCP first', async () => { await pass(); - sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App'); + sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); + expect(JSON.parse(portTestStub.firstCall.args[1]).protocols).to.deep.equal(['tcp']); + sinon.assert.notCalled(fluxUdpTestServer.FluxUdpTestServer); + }); + + it('tests UDP alone on the pass after a TCP pass that passed, and TCP after that', async () => { + await pass(); + portTestStub.callsFake(async (url, body) => { + const { ports: [port] } = JSON.parse(body); + return { data: { status: 'success', data: { udpAnswered: { [port]: fluxUdpTestServer.udpReply(udpToken).toString() } } } }; + }); + await pass(); + + expect(mapUpnpPortStub.secondCall.args[2]).to.deep.equal({ protocols: ['UDP'] }); + expect(JSON.parse(portTestStub.secondCall.args[1]).protocols).to.deep.equal(['udp']); + expect(fluxHttpTestServer.FluxHttpTestServer.prototype.listen.callCount, 'a TCP test server on the UDP pass').to.equal(1); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + + await pass(); + expect(mapUpnpPortStub.thirdCall.args[2]).to.deep.equal({ protocols: ['TCP'] }); + }); + + // A peer that does not know `protocols` tests TCP, finds it unmapped, and + // says the port failed: on a UDP pass that is no evidence at all. + it('reads nothing from a peer\'s TCP failure on a UDP pass, and tests UDP again with another peer', async () => { + mockDosState.testingProtocol = 'udp'; + portTestStub.callsFake(async (url, body) => { + const { ports: [port] } = JSON.parse(body); + return { data: { status: 'error', data: { message: `Flux Applications on 1.2.3.4:16127 are not available. Failed port: ${port}` } } }; + }); + + await pass(); + + expect(mockPortsNotWorking.size).to.equal(0); + expect(mockDosState.portFailureScore ?? 0).to.equal(0); + expect(mockDosState.nextTestingPort).to.equal(mockDosState.testingPort); + expect(mockDosState.testingProtocol).to.equal('udp'); + sinon.assert.calledWith(delayStub.lastCall, 15_000); + }); + + it('goes back to TCP after as many UDP passes as learn nothing', async () => { + mockDosState.testingProtocol = 'udp'; + portTestStub.resolves({ data: { status: 'error', data: { message: 'Failed port: 1' } } }); + + await pass(); + await pass(); + expect(mockDosState.testingProtocol, 'gave up early').to.equal('udp'); + await pass(); + + expect(mockDosState.testingProtocol).to.equal('tcp'); + expect(mockDosState.udpUnsettled).to.equal(0); + }); + + // A router refusing the UDP mapping is as broken as one refusing TCP. + it('counts a UDP mapping the router refused for its own reason as a router fault, and stays on UDP', async () => { + mockDosState.testingProtocol = 'udp'; + testMappingRefused(refused(606, true)); + + await pass(); + + expect(mockDosState.portFailureScore).to.equal(4); + expect(mockDosState.testingProtocol).to.equal('udp'); + sinon.assert.notCalled(portTestStub); + }); + + it('stays on TCP while its TCP test fails', async () => { + portTestStub.resolves({ data: { status: 'error', data: { message: 'Failed port: 1' } } }); + + await pass(); + + expect(mockPortsNotWorking.size).to.equal(1); + expect(mockDosState.testingProtocol ?? 'tcp').to.equal('tcp'); }); it('makes no test mapping, opens no port, and counts a failed check while the router has yet to remove a mapping of this node', async () => { @@ -553,8 +625,8 @@ describe('availabilityChecker tests', () => { await pass(); - expect(fluxNetworkHelper.allowTestPort.args).to.deep.equal([[mockDosState.testingPort, 'tcp'], [mockDosState.testingPort, 'udp']]); - expect(fluxNetworkHelper.deleteTestPortRule.args).to.deep.equal([[mockDosState.testingPort, 'tcp'], [mockDosState.testingPort, 'udp']]); + expect(fluxNetworkHelper.allowTestPort.args).to.deep.equal([[mockDosState.testingPort, 'tcp']]); + expect(fluxNetworkHelper.deleteTestPortRule.args).to.deep.equal([[mockDosState.testingPort, 'tcp']]); sinon.assert.notCalled(allowBoth); sinon.assert.notCalled(deleteBoth); }); @@ -681,7 +753,7 @@ describe('availabilityChecker tests', () => { const body = JSON.parse(portTestStub.firstCall.args[1]); expect(body.echo).to.equal(true); - expect(body.udp).to.equal(true); + expect(body.protocols).to.deep.equal(['tcp', 'udp']); sinon.assert.calledOnceWithExactly(udpServers[0].listen, mockDosState.testingPort); sinon.assert.calledOnceWithExactly(udpServers[0].allow, '192.168.1.200'); expect(udpServers[0].allow.calledBefore(portTestStub)).to.equal(true); diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index ef3a99396c..d117912ae0 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -1427,7 +1427,7 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', it('asks the peer to probe UDP, and opens a UDP test server on every port', async () => { await withPeers([answering('ours')]); - expect(JSON.parse(axios.post.firstCall.args[1]).udp).to.equal(true); + expect(JSON.parse(axios.post.firstCall.args[1]).protocols).to.deep.equal(['tcp', 'udp']); expect(udpServers).to.have.length(1); sinon.assert.calledWith(udpServers[0].listen, port); sinon.assert.calledOnce(udpServers[0].close); diff --git a/tests/unit/portTestEcho.test.js b/tests/unit/portTestEcho.test.js index aafef02bd3..4be1eb45a4 100644 --- a/tests/unit/portTestEcho.test.js +++ b/tests/unit/portTestEcho.test.js @@ -321,7 +321,7 @@ describe('checkAppAvailability probes the address that asked', () => { const answer = await ask( { - ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, udp: true, + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, protocols: ['tcp', 'udp'], }, HERE, ); @@ -341,7 +341,7 @@ describe('checkAppAvailability probes the address that asked', () => { const answer = await ask( { - ip: '::1', port: 16127, ports: [v6Port], pubKey: 'k', signature: 's', echo: true, udp: true, + ip: '::1', port: 16127, ports: [v6Port], pubKey: 'k', signature: 's', echo: true, protocols: ['tcp', 'udp'], }, '::1', ); @@ -352,6 +352,37 @@ describe('checkAppAvailability probes the address that asked', () => { expect(answer.data.udpAnswered).to.deep.equal({}); }); + it('probes UDP alone, reading nothing over TCP, for a caller that asks for UDP alone', async () => { + // Nothing listens on this port over TCP, so a TCP read would fail the test. + const udpServer = new FluxUdpTestServer(TOKEN); + await udpServer.listen(0, HERE); + udpServer.allow(HERE); + const udpOnly = udpServer.address().port; + const answer = await ask( + { + ip: HERE, port: 16127, ports: [udpOnly], pubKey: 'k', signature: 's', echo: true, protocols: ['udp'], + }, + HERE, + ); + udpServer.close(); + + expect(answer.status, 'the port was read over TCP').to.equal('success'); + expect(answer.data.udpAnswered[udpOnly]).to.include(TOKEN); + expect(answer.data).to.not.have.property('answered'); + }); + + it('refuses protocols other than tcp and udp', async () => { + const answer = await ask( + { + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, protocols: ['sctp'], + }, + HERE, + ); + + expect(answer.status).to.equal('error'); + expect(answer.data.message).to.include('tcp, udp or both'); + }); + it('probes no UDP for a caller that does not ask', async () => { const answer = await ask( { @@ -414,7 +445,7 @@ describe('checkAppAvailability probes the address that asked', () => { const started = Date.now(); const answer = await ask( { - ip: HERE, port: 16127, ports, pubKey: 'k', signature: 's', echo: true, udp: true, + ip: HERE, port: 16127, ports, pubKey: 'k', signature: 's', echo: true, protocols: ['tcp', 'udp'], }, HERE, ); From b53d89a0d33e8a8f6989ffae7d9f8483e03f3e5a Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 11:58:11 +0100 Subject: [PATCH 163/369] test(harness): a UPnP node maps TCP and UDP on alternate passes, and a router refusing UDP takes it out of service The gateway stub refuses every mapping of one protocol on request (protocol-fault). 3004 boots a node behind UPnP whose test mappings go TCP, UDP, TCP, one per pass, and whose router then refusing UDP mappings takes it out of service for its router with no TCP pass in between. 3003's first pass maps TCP alone. Co-Authored-By: Claude Opus 5.5 --- test-infra/external-http-stub/index.js | 24 ++++++ .../runner/framework/external-http-control.js | 13 +++ ...ses-it-is-out-of-service-until-it-stops.js | 6 +- .../tests/3004-a-port-is-proven-over-udp.js | 80 ++++++++++++++++++- 4 files changed, 119 insertions(+), 4 deletions(-) diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index cfd307968a..9aae072982 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -794,6 +794,12 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { soapFault(res, 728, 'NoPortMapsAvailable'); return; } + const protocolFault = gateway.protocolFaults?.[arg('NewProtocol').toUpperCase()]; + if (protocolFault) { + gateway.requests[gateway.requests.length - 1].refused = protocolFault; + soapFault(res, protocolFault, 'ActionFailed'); + return; + } gateway.mappings = gateway.mappings.filter((m) => !sameEntry(m)); gateway.mappings.push({ externalPort: Number(arg('NewExternalPort')), @@ -1092,6 +1098,24 @@ control.post('/upnp-gateway/:ip/delete-fault', (req, res) => { res.json({ ok: true }); }); +// Set the fault a node's gateway answers every mapping of one protocol with from +// now on (null: maps it as the spec says), keeping its table and the requests it +// has seen - a router that forwards TCP and refuses UDP. +control.post('/upnp-gateway/:ip/protocol-fault', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + const protocol = String(req.body?.protocol ?? '').toUpperCase(); + if (!['TCP', 'UDP'].includes(protocol)) { + res.status(400).json({ error: 'protocol must be TCP or UDP' }); + return; + } + gateway.protocolFaults = { ...gateway.protocolFaults, [protocol]: req.body?.code ?? null }; + res.json({ ok: true }); +}); + // Set how many mappings a node's gateway holds from now on (null: no limit), // keeping its table and the requests it has seen. control.post('/upnp-gateway/:ip/capacity', (req, res) => { diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index 2487fcafe6..d63d04db6f 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -220,6 +220,19 @@ export async function setUpnpRefuseDeletes(ip, count) { if (!res.ok) throw new Error(`setUpnpRefuseDeletes: ${JSON.stringify(res)}`); } +/** + * Set the fault a node's gateway answers every mapping of one protocol with from now on - a + * router that forwards TCP and refuses UDP. Keeps its table and the requests it has seen; each + * such answer carries `refused: `. + * @param {string} ip + * @param {'TCP'|'UDP'} protocol + * @param {number|null} code - null maps the protocol as the spec says + */ +export async function setUpnpProtocolFault(ip, protocol, code) { + const res = await post(`/upnp-gateway/${ip}/protocol-fault`, { protocol, code }); + if (!res.ok) throw new Error(`setUpnpProtocolFault: ${JSON.stringify(res)}`); +} + /** * Set the fault a node's gateway answers every GetSpecificPortMappingEntry with from now on, * whether or not it holds the entry. Keeps its table and the requests it has seen; each such diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js index 4d3631589c..ab8f997a66 100644 --- a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js @@ -120,7 +120,7 @@ describe('a node whose router refuses it is out of service until it stops', func await env?.teardown(); }); - it('tests its ports over TCP and UDP, and waits the long wait after a test that passed', async function () { + it('tests its port over one protocol a pass, TCP first, and waits the long wait after a test that passed', async function () { this.timeout(300000); let state = { requests: [] }; await waitFor(async () => { @@ -132,8 +132,8 @@ describe('a node whose router refuses it is out of service until it stops', func const made = testMappings(state.requests); const [first] = made; - expect(made.filter((r) => r.port === first.port).map((r) => r.protocol).sort(), 'both protocols of the port under test') - .to.deep.equal(['TCP', 'UDP']); + expect(made.filter((r) => r.port === first.port).map((r) => r.protocol), 'one protocol, TCP') + .to.deep.equal(['TCP']); // a failed test is retried within seconds; a passed one only after the long wait const quietFrom = Date.now(); diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index 348e757419..8c209d5c9d 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -9,7 +9,7 @@ import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { execInContainer, restartFluxos, setNodeApiPort } from '../framework/container.js'; -import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; +import { enableUpnpGateway, setUpnpProtocolFault, upnpGatewayState } from '../framework/external-http-control.js'; import { waitFor, waitForAppInstalled, waitForBootSettled } from '../framework/wait.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -56,6 +56,7 @@ const APPS_PORT_CHECK = { const PORT_TEST = { portTestMaxAttempts: 3 }; const PORTS_DOS_PREFIX = 'Ports tested not reachable from outside'; +const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; const appPinnedTo = async (name, port, index) => { await pushImage(name, 'v1'); @@ -225,6 +226,83 @@ describe('a node whose first peers cannot hear UDP replies', function () { }); }); +// A node behind UPnP maps one protocol a pass, TCP and UDP in turn, so a pass +// costs its router one mapping and one removal. A router refusing the UDP +// mapping is as broken as one refusing TCP. +describe('a node behind UPnP tests one protocol a pass', function () { + const NODE = 0; + let env; + let nodeIp; + dumpLogsOnFailure(() => env); + const testMappings = (requests) => requests.filter((r) => r.action === 'AddPortMapping' && r.description === 'Flux_Test_App'); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + // a passing test is followed by the next pass 20 s later, and a router fault + // reaches the DOS on its second count + nodeConfigOverrides: { + [NODE]: { + fluxapps: { + appsPortCheck: { + timeoutsMs: { + default: 20000, error: 2000, failure: 1000, dos: 5 * 60 * 1000, appError: 5000, + }, + thresholds: { dos: 8, portsHighEdge: 3, portsLowEdge: 1 }, + }, + }, + }, + }, + }); + nodeIp = subnet.nodeIp(NODE + 1); + const node = env.clients[NODE]; + await waitForBootSettled(node); + await enableUpnpGateway(nodeIp); + await setNodeApiPort(node.container, 16127); + await restartFluxos(node.container); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('maps TCP and UDP on alternate passes, one mapping a pass, and proves both', async function () { + this.timeout(600000); + let made = []; + await waitFor(async () => { + made = testMappings((await upnpGatewayState(nodeIp)).requests); + return made.length >= 3; + }, { timeout: 480000, interval: 5000, label: 'three passes behind UPnP' }); + + const [first, second, third] = made; + expect([first.protocol, second.protocol, third.protocol]).to.deep.equal(['TCP', 'UDP', 'TCP']); + expect(new Set([first.port, second.port, third.port]).size, 'one mapping a pass').to.equal(3); + expect(await udpCheck(env.clients[NODE])).to.equal('ok'); + }); + + it('is out of service, mapping UDP and nothing else, once its router refuses UDP mappings', async function () { + this.timeout(600000); + await setUpnpProtocolFault(nodeIp, 'UDP', 606); + const from = Date.now(); + + await waitFor(async () => (await env.clients[NODE].getDOSState()).data?.dosMessage === UPNP_DOS, { + timeout: 480000, interval: 5000, label: 'the node is out of service for its router', + }); + + const refusedSince = testMappings((await upnpGatewayState(nodeIp)).requests).filter((r) => Date.parse(r.at) >= from); + const lastTcp = refusedSince.findLastIndex((r) => r.protocol === 'TCP'); + const firstRefusal = refusedSince.findIndex((r) => r.refused === 606); + expect(firstRefusal, 'canary: the router refused a UDP mapping').to.be.at.least(0); + expect(lastTcp, 'a TCP pass after the router refused UDP').to.be.below(firstRefusal); + await setUpnpProtocolFault(nodeIp, 'UDP', null); + }); +}); + describe('a port closed over UDP, with the policy that counts it', function () { const BLOCKED = 0; let env; From 10ffbc7ad6730f594de41afa553d2e9cbad26e9f Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 12:02:34 +0100 Subject: [PATCH 164/369] fix(ports): a UDP test server's reply cap is per asked peer Probes forged from one asked peer's address use up that peer's replies and leave the next peer its own. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/utils/fluxUdpTestServer.js | 9 ++++++--- tests/unit/fluxUdpTestServer.test.js | 12 ++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/utils/fluxUdpTestServer.js b/ZelBack/src/services/utils/fluxUdpTestServer.js index ef177f14d5..5bf9f0c526 100644 --- a/ZelBack/src/services/utils/fluxUdpTestServer.js +++ b/ZelBack/src/services/utils/fluxUdpTestServer.js @@ -14,9 +14,10 @@ const UDP_PROBE_MAGIC = 'FLUXPT'; const UDP_PROBE_BYTES = 64; /** - * The most replies one test server sends while it listens. A peer sends a + * The most replies a test server sends to one asked peer. A peer sends a * handful of datagrams per port per attempt; this bounds what a flood of - * probes forged from the asked peer's address can make the server send. + * probes forged from the asked peer's address can make the server send, and + * a flood forged from one peer's address leaves the next peer its own replies. */ const MAX_REPLIES = 32; @@ -63,11 +64,13 @@ class FluxUdpTestServer { } /** - * The peer whose probes this server answers from now on; null answers nobody. + * The peer whose probes this server answers from now on, with a fresh reply + * cap; null answers nobody. * @param {string|null} ip */ allow(ip) { this.#allowed = ip ? ip.replace(/^::ffff:/i, '') : null; + this.#replies = 0; } /** diff --git a/tests/unit/fluxUdpTestServer.test.js b/tests/unit/fluxUdpTestServer.test.js index e498043c36..2097fcd422 100644 --- a/tests/unit/fluxUdpTestServer.test.js +++ b/tests/unit/fluxUdpTestServer.test.js @@ -90,6 +90,18 @@ describe('FluxUdpTestServer', () => { expect(await exchange(probe(), MAX_REPLIES + 8)).to.have.length(MAX_REPLIES); }); + // Probes forged from one asked peer's address use up that peer's replies, + // never the next peer's. + it('gives each peer it is told to answer a fresh reply cap', async () => { + server.allow('127.0.0.1'); + expect(await exchange(probe(), MAX_REPLIES), 'canary: the cap was used up').to.have.length(MAX_REPLIES); + expect(await exchange(probe())).to.deep.equal([]); + + server.allow('127.0.0.1'); + + expect(await exchange(probe())).to.have.length(1); + }); + it('refuses a token too long for a reply no larger than a probe', () => { expect(() => new FluxUdpTestServer('x'.repeat(UDP_PROBE_BYTES))).to.throw(); }); From ce1a9ecacf32291e21469b5d2a588bd68697a135 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 12:14:29 +0100 Subject: [PATCH 165/369] fix(ports): UDP heard on a port forwarded elsewhere goes with that reading, and the pre-install test reports UDP in health A peer that reads a port over TCP as something other than this node is reading a port forwarded elsewhere, so what it heard over UDP on that port describes the forward: it is kept out of the UDP tally and recorded as the notOurs decision's udpReadings (ours, silent, otherAnswer or notProbed). Two such peers refuse the install as notOurs. A pre-install proof over UDP reports UDP reachable in /flux/health, and a udpUnreachable refusal reports the port and the peers that heard nothing. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appNetwork/portManager.js | 32 ++++++++++++++++- tests/unit/portManager.test.js | 35 +++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 2b534c5df5..83aa9bdaaf 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -843,6 +843,21 @@ function udpPortsNotOurs(portsToTest, udpAnswered, token, proven) { return closed; } +/** + * What a peer heard over UDP on one port, read against our token: `ours`, + * `silent`, `otherAnswer`, or `notProbed` for a port it could not send to. + * @param {object} udpAnswered - port -> what that port replied, or null, from the peer + * @param {number} port + * @param {string} token + * @returns {string} + */ +function udpReadingOf(udpAnswered, port, token) { + const reply = udpAnswered[port] !== undefined ? udpAnswered[port] : udpAnswered[String(port)]; + if (reply === undefined) return 'notProbed'; + if (reply === null) return 'silent'; + return typeof reply === 'string' && reply.includes(token) ? 'ours' : 'otherAnswer'; +} + /** * Each peer whose UDP evidence still stands, with the first port it heard * nothing on that no peer has proven. @@ -1032,6 +1047,12 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { let tcpProven = false; // Ports our token came back from over UDP, from any peer. const udpProven = new Set(); + // Peer address -> what it heard over UDP on the port it read as not ours. + // A port answered over TCP by something else is a port forwarded elsewhere, + // so what the same peer heard over UDP describes that forward, not whether + // this node receives UDP: it goes with that peer's reading, never into the + // UDP tally. + const udpWhereNotOurs = new Map(); let sawNoUdpPeer = false; // Every peer already asked, so a redraw is ANOTHER peer rather than another // draw. Without this the picker can hand back the one just asked - which on @@ -1137,7 +1158,9 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { reportedPort = notOurs; } - if (udpAnswered && typeof udpAnswered === 'object') { + if (udpAnswered && typeof udpAnswered === 'object' && notOurs !== null) { + udpWhereNotOurs.set(askingIP, udpReadingOf(udpAnswered, notOurs, portTestToken)); + } else if (udpAnswered && typeof udpAnswered === 'object') { udpClosedPorts = udpPortsNotOurs(portsToTest, udpAnswered, portTestToken, udpProven); if (udpClosedPorts.length) { log.warn(`checkInstallingAppPortAvailable - ${askingIP} heard nothing of this node over UDP on port(s) ${udpClosedPorts.join(', ')}`); @@ -1149,6 +1172,7 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { if (tcpProven && everyUdpPortProven(portsToTest, udpProven)) { decision = { ok: true, reason: 'proven', port: null }; + fluxNetworkHelper.setUdpReachability({ reachable: true }); fluxEventBus.publish('ports:proven', { ...decision, ports: portsToTest, peer: askingIP }); break; } @@ -1194,6 +1218,10 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { port: refused, peers: [...disagreements.keys()], readings: Object.fromEntries(disagreements), + // what each of those peers heard over UDP on the same port, where it probed + udpReadings: Object.fromEntries([...disagreements.keys()] + .filter((peer) => udpWhereNotOurs.has(peer)) + .map((peer) => [peer, udpWhereNotOurs.get(peer)])), }; fluxEventBus.publish('ports:notOurs', decision); } @@ -1212,6 +1240,7 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { peers: [...disputed.keys()], readings: Object.fromEntries(disputed), }; + fluxNetworkHelper.setUdpReachability({ reachable: false, port: udpRefused, peers: decision.peers }); fluxEventBus.publish('ports:udpUnreachable', decision); } } @@ -1451,6 +1480,7 @@ module.exports = { portsInUseApi, portNotOurs, udpPortsNotOurs, + udpReadingOf, udpDisputes, everyUdpPortProven, refusedPort, diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index d117912ae0..b03b726542 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -1254,6 +1254,7 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', afterEach(() => { sinon.restore(); + fluxNetworkHelper.setUdpReachability(null); }); // One attempt per drawn peer: the observer is drawn first, so the attempt @@ -1549,6 +1550,33 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(result.readings).to.deep.equal({ '10.0.0.1': other, '10.0.0.2': other }); }); + // A port answered over TCP by something else is forwarded elsewhere; what the + // same peer heard over UDP describes that forward, not this node's UDP. + it('refuses as notOurs, not udpUnreachable, when the peers that hear nothing read TCP as someone else\'s', async () => { + const elsewhere = () => ({ + data: { + status: 'success', + data: { answered: { [port]: 'HTTP/1.1 200 OK\r\n\r\nsomebody else' }, udpAnswered: { [port]: null } }, + }, + }); + + const result = await withPeers([elsewhere, elsewhere, UNREACHABLE]); + + expect(result.reason).to.equal('notOurs'); + expect(result.udpReadings).to.deep.equal({ '10.0.0.1': 'silent', '10.0.0.2': 'silent' }); + expect(fluxNetworkHelper.getUdpReachability(), 'reported as this node\'s UDP').to.equal(null); + }); + + it('reports UDP reachable in health on a proof', async () => { + await withPeers([answering('ours')]); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + }); + + it('reports the refused port and its peers in health when two peers hear nothing over UDP', async () => { + await withPeers([answering(null), answering(null), UNREACHABLE]); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: false, port, peers: ['10.0.0.1', '10.0.0.2'] }); + }); + it('closes its UDP test servers when the test fails', async () => { fluxNetworkHelper.getLocalSocketAddress.resolves('1.2.3.4:16127'); fluxUdpTestServer.FluxUdpTestServer.callsFake(() => { @@ -1589,6 +1617,13 @@ describe('udpPortsNotOurs and udpDisputes', () => { expect(portManager.everyUdpPortProven([0], proven)).to.equal(true); }); + it('reads one port\'s UDP answer as ours, silent, another\'s answer, or not probed', () => { + expect(portManager.udpReadingOf({ 31000: ours }, 31000, TOKEN)).to.equal('ours'); + expect(portManager.udpReadingOf({ 31000: null }, 31000, TOKEN)).to.equal('silent'); + expect(portManager.udpReadingOf({ 31000: 'hello' }, 31000, TOKEN)).to.equal('otherAnswer'); + expect(portManager.udpReadingOf({}, 31000, TOKEN)).to.equal('notProbed'); + }); + it('keeps a peer as a witness only while a port it heard nothing on is unproven', () => { const disagreements = new Map([['a', [31000, 31001]], ['b', [31000]]]); expect(portManager.udpDisputes(disagreements, new Set([31000]))).to.deep.equal(new Map([['a', 31001]])); From f397faddc14183661c7642bae1cc42a1eaf0942b Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 13:04:57 +0100 Subject: [PATCH 166/369] test(ports): 50 silent ports prove nothing, whatever reaches a probe socket sharing the host Co-Authored-By: Claude Opus 5.5 --- tests/unit/portTestEcho.test.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit/portTestEcho.test.js b/tests/unit/portTestEcho.test.js index 4be1eb45a4..124e25b3e5 100644 --- a/tests/unit/portTestEcho.test.js +++ b/tests/unit/portTestEcho.test.js @@ -452,9 +452,12 @@ describe('checkAppAvailability probes the address that asked', () => { const elapsed = Date.now() - started; await Promise.all(servers.map((s) => new Promise((resolve) => s.close(resolve)))); + // Nothing proves a port. A probe can land on another probe's socket here - + // requester and peer share this host - so an answer can be a probe rather + // than silence; it is never our token. expect(answer.status).to.equal('success'); expect(Object.values(answer.data.udpAnswered)).to.have.length(50); - expect(Object.values(answer.data.udpAnswered).every((reply) => reply === null)).to.equal(true); + expect(Object.values(answer.data.udpAnswered).every((reply) => reply === null || !reply.includes(TOKEN))).to.equal(true); expect(elapsed, 'outlasted the requester\'s wait for a peer').to.be.below(config.fluxapps.portTestPeerTimeoutMs); }); From ade43e629936ebce0ac44045cee1b7374533e9c9 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 14:41:19 +0100 Subject: [PATCH 167/369] fix(ports): a pass whose TCP passes clears its failed ports even when UDP is retested; each pass publishes its verdict The first peer hearing nothing over UDP no longer ends the pass: its TCP result is handled as any other pass's, clearing the failed ports and this check's DOS on a TCP pass, and only the port and the wait are the UDP retest's. Each availability pass publishes availability:pass: the port and protocols it tested, the TCP and UDP outcomes, the failed ports left, whether it lifted its DOS, and whether UDP is retested. 3004 blocks a UDP-blocked node's inbound TCP until its check records failed ports, unblocks it, and asserts the pass whose TCP passes and whose first peer hears nothing over UDP clears them and schedules the UDP retest. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/availabilityChecker.js | 30 ++++++++-- test-infra/runner/framework/node-client.js | 3 + .../tests/3004-a-port-is-proven-over-udp.js | 55 +++++++++++++++++++ tests/unit/availabilityChecker.test.js | 31 +++++++++++ 4 files changed, 113 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index e71aa4cd3a..2647f06421 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -13,6 +13,7 @@ const fluxUdpTestServer = require('../utils/fluxUdpTestServer'); const portManager = require('../appNetwork/portManager'); const registryManager = require('../appDatabase/registryManager'); const policyStore = require('../policyStore'); +const fluxEventBus = require('../utils/fluxEventBus'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -536,6 +537,8 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // A pass that does not test TCP reads nothing from the peer's TCP answer: a // peer that does not know `protocols` tests TCP anyway, and finds it unmapped. let portTestFailed = tcpTested && responseStatus === 'error'; + let tcpOutcome = 'notTested'; + if (tcpTested) tcpOutcome = portTestFailed ? 'unreachable' : 'passed'; // A peer that read the port: only our token says this node is what // answered. Anything else holding the port leaves it as unusable here as @@ -543,6 +546,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (tcpTested && !portTestFailed && answered && portManager.portNotOurs([dosState.testingPort], answered, portTestToken) !== null) { log.warn(`checkMyAppsAvailability - Port ${dosState.testingPort} answered ${remoteIp}:${remotePort} as something other than this node`); portTestFailed = true; + tcpOutcome = 'notOurs'; } // Over UDP, our token heard by one peer proves the port; silence heard by @@ -554,14 +558,12 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin dosState.udpSuspect = null; fluxNetworkHelper.setUdpReachability({ reachable: true }); } else if (udp === 'silent' && !udpSuspect) { + // The pass's TCP result is handled as any other's; only the port and the + // wait are the retest's. log.warn(`checkMyAppsAvailability - ${remoteIp}:${remotePort} heard nothing of this node on port ${dosState.testingPort} over UDP; asking another peer`); // eslint-disable-next-line no-param-reassign dosState.udpSuspect = { port: dosState.testingPort, peer: remoteSocketAddress, retests: 0 }; - // eslint-disable-next-line no-param-reassign - dosState.nextTestingPort = dosState.testingPort; - await waitForNextPass(timeouts.failure); - setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); - return; + retestUdp = true; } else if (udp === 'silent') { const peers = [extractIp(udpSuspect.peer), remoteIp]; log.error(`checkMyAppsAvailability - Port ${dosState.testingPort} does not answer this node over UDP, as heard by ${peers.join(' and ')}` @@ -588,6 +590,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin let waitMs = 0; let passed = false; + let dosReleased = false; if (portTestFailed && portsNotWorking.size < thresholds.portsHighEdge) { portsNotWorking.add(dosState.testingPort); @@ -625,7 +628,10 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // eslint-disable-next-line no-param-reassign dosState.originalPortFailed = null; // a router refusing others' mappings is not cleared by one that took this test's - if (!routerRefused) releaseOwnDos(dosState); + if (!routerRefused) { + releaseOwnDos(dosState); + dosReleased = true; + } waitMs = timeouts.default; } @@ -658,6 +664,18 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isUpnp && routerFaulting) { waitMs = Math.min(waitMs, dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); } + fluxEventBus.publish('availability:pass', { + port: dosState.testingPort, + protocols, + peer: remoteIp, + tcp: tcpOutcome, + udp: udpTested ? udp : 'notTested', + portsNotWorking: portsNotWorking.size, + dosReleased, + udpRetest: retestUdp, + nextPort: dosState.nextTestingPort, + waitMs, + }); await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 62f509a296..16ef8cd534 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -233,6 +233,9 @@ export function nodeClient(nodeNum) { 'ports:unproven', 'ports:proven', 'ports:udpUnreachable', + // One availability pass's verdict: the port and protocols it tested, + // each outcome, and what it cleared and scheduled. + 'availability:pass', 'discovery:started', 'peers:added', 'peers:accepting', diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index 8c209d5c9d..a510e33416 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -303,6 +303,61 @@ describe('a node behind UPnP tests one protocol a pass', function () { }); }); +// A pass whose TCP passes clears what the TCP failures before it left, even +// when the first peer hears nothing over UDP and the port is retested over UDP. +describe('a node whose TCP comes back while its UDP is blocked', function () { + const NODE = 0; + let env; + dumpLogsOnFailure(() => env); + // Inbound TCP connections to the node, bar its FluxOS ports, dropped on its fleet device. + const tcpBlock = (op) => { + const ip = subnet.nodeIp(NODE + 1); + return `dev=$(ip -o -4 addr show | awk -v ip=${ip} '{ split($4, a, "/"); if (a[1] == ip) print $2 }'); ` + + `iptables -w ${op} INPUT -i "$dev" -p tcp -m conntrack --ctstate NEW -m multiport ! --dports 16100:16299 -j DROP`; + }; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: { [NODE]: 'udp-blocked' }, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + // failed ports never outnumber the low edge, so a passing pass clears them all + nodeConfigOverrides: { + [NODE]: { fluxapps: { appsPortCheck: { ...APPS_PORT_CHECK, thresholds: { dos: 100, portsHighEdge: 2, portsLowEdge: 2 } } } }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('clears its failed ports on the pass whose TCP passes and whose first peer hears nothing over UDP', async function () { + this.timeout(600000); + const client = env.clients[NODE]; + const blockedFrom = client.getLastEventId(); + const blocked = await execInContainer(client.container, tcpBlock('-I')); + expect(blocked.exitCode, blocked.output).to.equal(0); + const failing = await client.waitForEvent('availability:pass', (d) => d.tcp === 'unreachable' && d.portsNotWorking >= 1, 300000, { afterId: blockedFrom }); + expect(failing.data.udp, 'UDP is probed only once TCP has passed').to.equal('unknown'); + + const unblockedFrom = client.getLastEventId(); + const unblocked = await execInContainer(client.container, tcpBlock('-D')); + expect(unblocked.exitCode, unblocked.output).to.equal(0); + const recovered = await client.waitForEvent('availability:pass', (d) => d.tcp === 'passed', 300000, { afterId: unblockedFrom }); + + expect(recovered.data.udp, 'canary: the first peer heard nothing over UDP').to.equal('silent'); + expect(recovered.data.portsNotWorking, 'failed ports left for the retest to clear').to.equal(0); + expect(recovered.data.dosReleased).to.equal(true); + expect(recovered.data.udpRetest).to.equal(true); + expect(recovered.data.nextPort).to.equal(recovered.data.port); + }); +}); + describe('a port closed over UDP, with the policy that counts it', function () { const BLOCKED = 0; let env; diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index f2551ef875..6128eebe46 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -18,6 +18,7 @@ const fluxUdpTestServer = require('../../ZelBack/src/services/utils/fluxUdpTestS const portManager = require('../../ZelBack/src/services/appNetwork/portManager'); const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); const policyStore = require('../../ZelBack/src/services/policyStore'); +const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); const signer = { pubKey: 'pubkey', sign: () => 'signature' }; // the check with a node that can sign its port test @@ -826,6 +827,36 @@ describe('availabilityChecker tests', () => { sinon.assert.calledWith(networkStateService.getRandomExternalObserver.secondCall, '192.168.1.100:16127', { exclude: [PEERS[0]] }); }); + // A pass whose TCP passes is a passing pass for TCP, whatever UDP needs next. + it('clears its failed ports and lifts its DOS on a TCP pass whose first peer heard nothing over UDP, and still retests UDP', async () => { + const publish = sinon.spy(fluxEventBus, 'publish'); + mockPortsNotWorking.add(1234); + mockDosState.portFailureScore = 8; + mockDosState.dosMessage = 'Ports tested not reachable from outside, DMZ or UPNP required! All ports that have failed: [1234]'; + mockDosState.dosStateValue = 100; + portTestStub.callsFake(reading('ours', null)); + + await pass(); + + expect(mockPortsNotWorking.size, 'failed ports left until the retest').to.equal(0); + expect(mockDosState.dosMessage, 'its DOS left until the retest').to.equal(null); + expect(mockDosState.nextTestingPort).to.equal(mockDosState.testingPort); + sinon.assert.calledWith(delayStub.lastCall, 15_000); + sinon.assert.calledWithMatch(publish, 'availability:pass', { + port: mockDosState.testingPort, tcp: 'passed', udp: 'silent', portsNotWorking: 0, dosReleased: true, udpRetest: true, + }); + }); + + it('publishes every pass\'s verdict', async () => { + const publish = sinon.spy(fluxEventBus, 'publish'); + + await pass(); + + sinon.assert.calledWithMatch(publish, 'availability:pass', { + protocols: ['tcp', 'udp'], peer: '192.168.1.200', tcp: 'passed', udp: 'proven', udpRetest: false, + }); + }); + it('takes our token heard by a second peer over the first peer\'s silence', async () => { portTestStub.onFirstCall().callsFake(reading('ours', null)); From 78b7a76094075a0dab660d4abf7295ded17b4179 Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 14:42:57 +0100 Subject: [PATCH 168/369] feat(ports): the signed policy can turn the pre-install refusal over UDP off udpinstallrefusaloff published as true makes a port two peers heard nothing on over UDP install as udpUnproven instead of being refused, its UDP still reported in /flux/health. Absent or anything else, the refusal stands, so it changes nothing until it is published. The policy stub publishes the document; 3004's policy fleet carries it, and a UDP-blocked node there installs as udpUnproven. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appNetwork/portManager.js | 44 ++++++++++++++----- test-infra/external-http-stub/index.js | 5 +++ .../tests/3004-a-port-is-proven-over-udp.js | 21 +++++++-- tests/unit/portManager.test.js | 21 +++++++++ 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 83aa9bdaaf..7f02a99c5f 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -18,6 +18,7 @@ const { Privilege, authOf } = require('../utils/privileges'); const fluxCaching = require('../utils/cacheManager'); const globalState = require('../utils/globalState'); const fluxEventBus = require('../utils/fluxEventBus'); +const policyStore = require('../policyStore'); const { nodeSigner } = require('../utils/nodeSigner'); // Global cache for failed nodes @@ -735,6 +736,22 @@ async function askSiblingsForHeldPort(appPorts, localSocketAddress) { return null; } +/** + * The signed policy document that turns the pre-install refusal over UDP off. + * Published as `true`, a port two peers heard nothing on over UDP installs as + * `udpUnproven`, its UDP still reported in /flux/health; absent or anything + * else, the refusal stands. + */ +const UDP_INSTALL_REFUSAL_OFF_DOCUMENT = 'udpinstallrefusaloff'; + +/** + * Whether the signed policy has turned the pre-install refusal over UDP off. + * @returns {boolean} + */ +function udpInstallRefusalOff() { + return policyStore.getDocument(UDP_INSTALL_REFUSAL_OFF_DOCUMENT) === true; +} + /** * How many independent peers must agree before a port test refuses an install. * Not config: it is a property of what counts as evidence, not a tuning knob, @@ -1231,17 +1248,22 @@ async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { const udpRefused = refusedPort(disputed, PORT_TEST_CORROBORATION); if (udpRefused !== null) { const readings = [...disputed.entries()].map(([peer, port]) => `${peer} on port ${port}`); - log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} at this address does not answer this node over UDP, as heard by ` - + `${disputed.size} peers (${readings.join(', ')}) and proven by none of the ${asked.length} asked. Installation aborted.`); - decision = { - ok: false, - reason: 'udpUnreachable', - port: udpRefused, - peers: [...disputed.keys()], - readings: Object.fromEntries(disputed), - }; - fluxNetworkHelper.setUdpReachability({ reachable: false, port: udpRefused, peers: decision.peers }); - fluxEventBus.publish('ports:udpUnreachable', decision); + fluxNetworkHelper.setUdpReachability({ reachable: false, port: udpRefused, peers: [...disputed.keys()] }); + if (udpInstallRefusalOff()) { + log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} does not answer this node over UDP, as heard by ` + + `${readings.join(', ')}; the network policy has the refusal over UDP off, so the install goes ahead`); + } else { + log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} at this address does not answer this node over UDP, as heard by ` + + `${disputed.size} peers (${readings.join(', ')}) and proven by none of the ${asked.length} asked. Installation aborted.`); + decision = { + ok: false, + reason: 'udpUnreachable', + port: udpRefused, + peers: [...disputed.keys()], + readings: Object.fromEntries(disputed), + }; + fluxEventBus.publish('ports:udpUnreachable', decision); + } } } diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 9aae072982..dfb67583a2 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -411,6 +411,8 @@ const state = { // Whether a port closed over UDP counts toward the availability check's DOS. null is // the document absent from the published bundle, which is the default. udpPortDos: null, + // Whether the pre-install refusal over UDP is off. null is the document absent. + udpInstallRefusalOff: null, latestRelease: { tag_name: 'v0.0.0', name: 'stub-release' }, geolocation: {}, // The syncthing the node image ships, read from the repository the image was built @@ -547,6 +549,7 @@ function resignPolicy({ bumpSeq = true } = {}) { tamperingblockednodes: state.tamperingBlocklist, enterprisenodes: state.enterpriseNodes, ...(state.udpPortDos === null ? {} : { udpportdos: state.udpPortDos }), + ...(state.udpInstallRefusalOff === null ? {} : { udpinstallrefusaloff: state.udpInstallRefusalOff }), }, // Content-addressed, as the real publisher does it: the plain name is mutable and means // "the current table", so the bundle names the hash a fetch should produce. Derived from @@ -1296,6 +1299,7 @@ control.post('/policy', (req, res) => { tamperingblockednodes: 'tamperingBlocklist', enterprisenodes: 'enterpriseNodes', udpportdos: 'udpPortDos', + udpinstallrefusaloff: 'udpInstallRefusalOff', }; for (const [name, value] of Object.entries(body.documents)) { if (!map[name]) return res.status(400).json({ error: `unknown document ${name}` }); @@ -1344,6 +1348,7 @@ control.post('/reset', (req, res) => { state.tamperingBlocklist = []; state.enterpriseNodes = {}; state.udpPortDos = null; + state.udpInstallRefusalOff = null; state.latestRelease = { tag_name: 'v0.0.0', name: 'stub-release' }; state.geolocation = {}; artifacts.clear(); diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index a510e33416..35511c6aaa 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -358,8 +358,10 @@ describe('a node whose TCP comes back while its UDP is blocked', function () { }); }); -describe('a port closed over UDP, with the policy that counts it', function () { +describe('a port closed over UDP, under the policy that counts it and turns the install refusal off', function () { const BLOCKED = 0; + // blocked too, on the default thresholds, so it stays in service to install + const INSTALLS = 1; let env; dumpLogsOnFailure(() => env); @@ -368,8 +370,8 @@ describe('a port closed over UDP, with the policy that counts it', function () { env = await createTestEnv({ hookCtx: this, nodes: 4, - networkShapes: { [BLOCKED]: 'udp-blocked' }, - policy: { documents: { udpportdos: true } }, + networkShapes: { [BLOCKED]: 'udp-blocked', [INSTALLS]: 'udp-blocked' }, + policy: { documents: { udpportdos: true, udpinstallrefusaloff: true } }, configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, // the first port failure is walked, the next counts, and one count is the threshold nodeConfigOverrides: { @@ -384,6 +386,19 @@ describe('a port closed over UDP, with the policy that counts it', function () { await env?.teardown(); }); + // The switch that turns the refusal off without a release. + it('installs on a port two peers hear nothing on over UDP, as udpUnproven', async function () { + this.timeout(420000); + const client = env.clients[INSTALLS]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo('udprefusaloffapp', 31147, INSTALLS)); + + const unproven = await client.waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); + expect(unproven.data.reason).to.equal('udpUnproven'); + expect(unproven.data.peers, 'canary: peers heard nothing over UDP').to.have.length.at.least(2); + await waitForAppInstalled(client, 'udprefusaloffapp', 300000, { afterId: after }); + }); + it('takes the node out of service once the failures it confirms reach the threshold', async function () { this.timeout(600000); await waitFor(async () => (await env.clients[BLOCKED].getDOSState()).data?.dosMessage?.startsWith(PORTS_DOS_PREFIX) === true, { diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index b03b726542..daf6d4559e 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -13,6 +13,7 @@ const networkStateService = require('../../ZelBack/src/services/networkStateServ const { EventEmitter } = require('node:events'); const fluxHttpTestServer = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); const fluxUdpTestServer = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); +const policyStore = require('../../ZelBack/src/services/policyStore'); const { requireMongo } = require('./dbTestHelper'); const appQueryService = require('../../ZelBack/src/services/appQuery/appQueryService'); const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); @@ -1572,6 +1573,26 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); }); + // A kill switch, not an opt-in: absent, the refusal stands. + it('installs as udpUnproven, still reporting UDP in health, when the policy turns the refusal over UDP off', async () => { + sinon.stub(policyStore, 'getDocument').withArgs('udpinstallrefusaloff').returns(true); + + const result = await withPeers([answering(null), answering(null), UNREACHABLE]); + + expect(result.ok).to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + expect(result.peers).to.deep.equal(['10.0.0.1', '10.0.0.2']); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: false, port, peers: ['10.0.0.1', '10.0.0.2'] }); + }); + + it('refuses when the policy carries the switch as anything but true', async () => { + sinon.stub(policyStore, 'getDocument').withArgs('udpinstallrefusaloff').returns(false); + + const result = await withPeers([answering(null), answering(null), UNREACHABLE]); + + expect(result.reason).to.equal('udpUnreachable'); + }); + it('reports the refused port and its peers in health when two peers hear nothing over UDP', async () => { await withPeers([answering(null), answering(null), UNREACHABLE]); expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: false, port, peers: ['10.0.0.1', '10.0.0.2'] }); From 215fc218554c48fcbfcc8d3bb5dfb30a3dbdbeae Mon Sep 17 00:00:00 2001 From: David White Date: Wed, 21 Oct 2026 15:49:50 +0100 Subject: [PATCH 169/369] test(ports): an availability pass's branch is a tally, not an event The bus streams facts and tallies cadence, and the event buffer is shared by every test: each pass now counts availability:pass by its TCP and UDP outcomes, and a pass whose TCP passes and whose first peer hears nothing over UDP counts availability:firstUdpSilence as cleared or notCleared. 3004 reads both from /flux/testcounters. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/availabilityChecker.js | 25 ++++++------------- test-infra/runner/framework/node-client.js | 3 --- .../tests/3004-a-port-is-proven-over-udp.js | 23 +++++++++-------- tests/unit/availabilityChecker.test.js | 15 +++++------ 4 files changed, 27 insertions(+), 39 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index 2647f06421..6960f118a1 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -552,6 +552,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // Over UDP, our token heard by one peer proves the port; silence heard by // one peer is tested again by another, and counts once two have heard it. let retestUdp = false; + let firstUdpSilence = false; const udp = !udpTested || portTestFailed ? 'unknown' : udpVerdict(udpAnswered, dosState.testingPort, portTestToken); if (udp === 'proven') { // eslint-disable-next-line no-param-reassign @@ -564,6 +565,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // eslint-disable-next-line no-param-reassign dosState.udpSuspect = { port: dosState.testingPort, peer: remoteSocketAddress, retests: 0 }; retestUdp = true; + firstUdpSilence = true; } else if (udp === 'silent') { const peers = [extractIp(udpSuspect.peer), remoteIp]; log.error(`checkMyAppsAvailability - Port ${dosState.testingPort} does not answer this node over UDP, as heard by ${peers.join(' and ')}` @@ -590,7 +592,6 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin let waitMs = 0; let passed = false; - let dosReleased = false; if (portTestFailed && portsNotWorking.size < thresholds.portsHighEdge) { portsNotWorking.add(dosState.testingPort); @@ -628,10 +629,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin // eslint-disable-next-line no-param-reassign dosState.originalPortFailed = null; // a router refusing others' mappings is not cleared by one that took this test's - if (!routerRefused) { - releaseOwnDos(dosState); - dosReleased = true; - } + if (!routerRefused) releaseOwnDos(dosState); waitMs = timeouts.default; } @@ -664,18 +662,11 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isUpnp && routerFaulting) { waitMs = Math.min(waitMs, dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); } - fluxEventBus.publish('availability:pass', { - port: dosState.testingPort, - protocols, - peer: remoteIp, - tcp: tcpOutcome, - udp: udpTested ? udp : 'notTested', - portsNotWorking: portsNotWorking.size, - dosReleased, - udpRetest: retestUdp, - nextPort: dosState.nextTestingPort, - waitMs, - }); + // Which branch each pass took, as a tally: the passes run too often to stream. + fluxEventBus.count('availability:pass', tcpOutcome, udpTested ? udp : 'notTested'); + if (firstUdpSilence) { + fluxEventBus.count('availability:firstUdpSilence', tcpOutcome, portsNotWorking.size === 0 ? 'cleared' : 'notCleared'); + } await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 16ef8cd534..62f509a296 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -233,9 +233,6 @@ export function nodeClient(nodeNum) { 'ports:unproven', 'ports:proven', 'ports:udpUnreachable', - // One availability pass's verdict: the port and protocols it tested, - // each outcome, and what it cleared and scheduled. - 'availability:pass', 'discovery:started', 'peers:added', 'peers:accepting', diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js index 35511c6aaa..2170daa098 100644 --- a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -336,25 +336,28 @@ describe('a node whose TCP comes back while its UDP is blocked', function () { await env?.teardown(); }); + // Which branch a pass took is a tally on /flux/testcounters, never a stream. it('clears its failed ports on the pass whose TCP passes and whose first peer hears nothing over UDP', async function () { this.timeout(600000); const client = env.clients[NODE]; - const blockedFrom = client.getLastEventId(); + const tally = async (name, ...path) => path.reduce((node, key) => node?.[key], (await client.getTestCounters())[name]) || 0; + const blocked = await execInContainer(client.container, tcpBlock('-I')); expect(blocked.exitCode, blocked.output).to.equal(0); - const failing = await client.waitForEvent('availability:pass', (d) => d.tcp === 'unreachable' && d.portsNotWorking >= 1, 300000, { afterId: blockedFrom }); - expect(failing.data.udp, 'UDP is probed only once TCP has passed').to.equal('unknown'); + const unreachableBefore = await tally('availability:pass', 'unreachable', 'unknown'); + await waitFor(async () => (await tally('availability:pass', 'unreachable', 'unknown')) > unreachableBefore, { + timeout: 300000, interval: 2000, label: 'a pass whose TCP test failed', + }); - const unblockedFrom = client.getLastEventId(); const unblocked = await execInContainer(client.container, tcpBlock('-D')); expect(unblocked.exitCode, unblocked.output).to.equal(0); - const recovered = await client.waitForEvent('availability:pass', (d) => d.tcp === 'passed', 300000, { afterId: unblockedFrom }); + await waitFor(async () => (await tally('availability:firstUdpSilence', 'passed', 'cleared')) >= 1 + || (await tally('availability:firstUdpSilence', 'passed', 'notCleared')) >= 1, { + timeout: 300000, interval: 2000, label: 'a pass whose TCP passed and whose first peer heard nothing over UDP', + }); - expect(recovered.data.udp, 'canary: the first peer heard nothing over UDP').to.equal('silent'); - expect(recovered.data.portsNotWorking, 'failed ports left for the retest to clear').to.equal(0); - expect(recovered.data.dosReleased).to.equal(true); - expect(recovered.data.udpRetest).to.equal(true); - expect(recovered.data.nextPort).to.equal(recovered.data.port); + expect(await tally('availability:firstUdpSilence', 'passed', 'notCleared'), 'failed ports left for the retest to clear').to.equal(0); + expect(await tally('availability:firstUdpSilence', 'passed', 'cleared')).to.be.at.least(1); }); }); diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 6128eebe46..47795d4416 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -829,7 +829,7 @@ describe('availabilityChecker tests', () => { // A pass whose TCP passes is a passing pass for TCP, whatever UDP needs next. it('clears its failed ports and lifts its DOS on a TCP pass whose first peer heard nothing over UDP, and still retests UDP', async () => { - const publish = sinon.spy(fluxEventBus, 'publish'); + const count = sinon.spy(fluxEventBus, 'count'); mockPortsNotWorking.add(1234); mockDosState.portFailureScore = 8; mockDosState.dosMessage = 'Ports tested not reachable from outside, DMZ or UPNP required! All ports that have failed: [1234]'; @@ -842,19 +842,16 @@ describe('availabilityChecker tests', () => { expect(mockDosState.dosMessage, 'its DOS left until the retest').to.equal(null); expect(mockDosState.nextTestingPort).to.equal(mockDosState.testingPort); sinon.assert.calledWith(delayStub.lastCall, 15_000); - sinon.assert.calledWithMatch(publish, 'availability:pass', { - port: mockDosState.testingPort, tcp: 'passed', udp: 'silent', portsNotWorking: 0, dosReleased: true, udpRetest: true, - }); + sinon.assert.calledWithExactly(count, 'availability:firstUdpSilence', 'passed', 'cleared'); }); - it('publishes every pass\'s verdict', async () => { - const publish = sinon.spy(fluxEventBus, 'publish'); + it('tallies the branch each pass took', async () => { + const count = sinon.spy(fluxEventBus, 'count'); await pass(); - sinon.assert.calledWithMatch(publish, 'availability:pass', { - protocols: ['tcp', 'udp'], peer: '192.168.1.200', tcp: 'passed', udp: 'proven', udpRetest: false, - }); + sinon.assert.calledWithExactly(count, 'availability:pass', 'passed', 'proven'); + sinon.assert.neverCalledWith(count, 'availability:firstUdpSilence'); }); it('takes our token heard by a second peer over the first peer\'s silence', async () => { From fe7076b9239687906d94a5e3d9b23c7426a8e80e Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:05:36 +0100 Subject: [PATCH 170/369] fix(apps): boot holds an app whose location it cannot read, and decides it once it can An app boot finds stopped is started when this node's location record of it is valid and removed when it has expired. A record that cannot be read answers neither, so the app is held in globalState.unconfirmedApps: the reconciler starts none of its components, the election neither promotes nor stands it down, the announcement leaves it out, and backup, restore and redeploy refuse it. The record is read again every 15 s; once it answers the app is released and started, or removed without telling the network, which has already dropped the record. The announcement exclusion matters once the database answers again: naming the app would renew a record that may have expired, and the boot check would then read it as valid and start the app beside the copy placed elsewhere. An app whose removal is pending is likewise neither elected, backed up nor restored: each would set its folder's type or start it. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 35 +++++ .../appLifecycle/appStartupManager.js | 107 +++++++++++--- .../services/appMessaging/peerNotification.js | 7 +- .../services/appMonitoring/appReconciler.js | 3 + ZelBack/src/services/utils/fluxEventBus.js | 3 + ZelBack/src/services/utils/globalState.js | 9 ++ ...08-boot-holds-an-app-it-cannot-confirm.js} | 0 tests/unit/advancedWorkflows.test.js | 131 ++++++++++++++++++ tests/unit/appReconciler.test.js | 10 ++ tests/unit/appStartupManager.test.js | 118 ++++++++++++++-- tests/unit/peerNotification.test.js | 40 +++++- 11 files changed, 427 insertions(+), 36 deletions(-) rename test-infra/runner/tests/{3308-boot-removes-an-app-it-cannot-confirm.js => 3308-boot-holds-an-app-it-cannot-confirm.js} (100%) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 184d5454fa..45492ac790 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -1473,6 +1473,12 @@ async function mayTearDownToRebuild(appSpecs, res) { } }; + // A rebuilt app starts, and boot has yet to decide whether this one may. + if (globalState.unconfirmedApps.has(appSpecs.name)) { + report(`${appSpecs.name} is held until this node confirms it still holds it, redeploy deferred`); + return false; + } + const pinned = Array.isArray(appSpecs.nodes) ? appSpecs.nodes : []; if (pinned.length) { const collateral = await generalService.obtainNodeCollateralInformation().catch(() => null); @@ -2745,6 +2751,18 @@ async function startOperatorStarted(identifier, appId) { * @param {object} req - Request object * @param {object} res - Response object */ +/** + * Refuses a task that starts an app this node is removing, or has not yet confirmed + * it still holds. + * @param {string} appName + * @returns {void} + * @throws {Error} When the app is held. + */ +function refuseHeldApp(appName) { + if (globalState.departingApps.has(appName)) throw new Error(`Refused: ${appName} is being removed from this node`); + if (globalState.unconfirmedApps.has(appName)) throw new Error(`Refused: ${appName} is held until this node confirms it still holds it`); +} + async function appendBackupTask(req, res) { let appname; let backup; @@ -2769,6 +2787,8 @@ async function appendBackupTask(req, res) { if (hasTrueBackup === false) { throw new Error('No backup jobs...'); } + // a backup ends by starting the app + refuseHeldApp(appname); // The claim, before any awaited work: a second request for the same app // finds it taken here rather than passing an emptied check and racing to // the archive alongside the first. Last in this synchronous block, so a @@ -3005,6 +3025,8 @@ async function appendRestoreTask(req, res) { if (hasTrueRestore === false) { throw new Error('No restore jobs...'); } + // a restore ends by starting the app + refuseHeldApp(appname); // The claim, before any awaited work: a second request for the same app // finds it taken here rather than passing an emptied check and racing to // the clear alongside the first. Last in this synchronous block, so a @@ -5122,6 +5144,19 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-continue continue; } + // An app this node is removing, or has not yet confirmed it still holds, is + // neither promoted nor stood down: either would set its folder's type, and + // the reconciler starts none of its components. + if (globalStateParam.departingApps.has(installedApp.name)) { + fluxEventBus.count('masterSlave:decision', installedApp.name, 'skippedDeparting'); + // eslint-disable-next-line no-continue + continue; + } + if (globalStateParam.unconfirmedApps.has(installedApp.name)) { + fluxEventBus.count('masterSlave:decision', installedApp.name, 'skippedUnconfirmed'); + // eslint-disable-next-line no-continue + continue; + } // Every g: component of the app. The election decides the app: its primary // runs all of them on one node, and every other holder runs none. Checked // for all g: mode apps, not just those in cache with restarted flag - the diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 17b27de5b7..ba24dfa56e 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -23,6 +23,7 @@ const nodeConfirmationService = require('../nodeConfirmationService'); const { localAppsInformation, SIGTERM_EXPIRY_MS, RUNNING_EXPIRY_MS } = require('../utils/appConstants'); const { getNonGComponentIdentifiers, parseContainerName, appHasValidLocationOnNode } = require('../utils/appUtilities'); const { RemovalOutcome } = require('../utils/removalOutcome'); +const { untilSucceeds, BOOT_STEP_RETRY_MS } = require('../utils/untilSucceeds'); const SYNC_TIMEOUT_MS = config.system.bootSyncTimeoutMs ?? 300000; @@ -116,25 +117,84 @@ async function openInstalledAppPorts() { /** - * Removes an app boot may not start. A removal that does not finish holds the app as - * departing - never started by the reconciler, never announced as running - and is - * run again until it does. + * Removes an app whose location this node no longer holds. The network has dropped + * the record, so the removal tells it nothing. A removal that does not finish holds + * the app as departing - never started by the reconciler, never announced as + * running - and is run again until it does. * @param {string} appName - * @param {boolean} announce Whether the removal is told to the network. * @returns {Promise} Whether the app is gone now. */ -async function removeAppBootMayNotStart(appName, announce) { - const outcome = await appUninstaller.removeAppLocally(appName, null, true, true, announce); +async function removeAppBootMayNotStart(appName) { + const outcome = await appUninstaller.removeAppLocally(appName, null, true, true, false); if (outcome === RemovalOutcome.REMOVED || outcome === RemovalOutcome.NOT_INSTALLED) return true; - appUninstaller.finishRemoval(appName, true, announce); + appUninstaller.finishRemoval(appName, true, false); return false; } + +/** + * Hands the components boot starts to the reconciler. + * @param {string[]} componentsToStart + * @returns {void} + */ +function startAtBoot(componentsToStart) { + // the reconciler actuates once the boot gate opens (daemon/DB ready) and applies + // desired-state + restart policy + // eslint-disable-next-line no-restricted-syntax + for (const identifier of componentsToStart) { + appReconciler.enqueue(identifier); + log.info(`appStartupManager - Queued ${identifier} for reconcile`); + } +} + +/** + * Whether this node holds an unexpired location record for an app. + * @param {string} appName + * @param {string} localSocketAddr + * @returns {Promise} + * @throws {Error} When the records cannot be read. + */ +async function readLocation(appName, localSocketAddr) { + const valid = await appHasValidLocationOnNode(appName, localSocketAddr); + if (valid === null) throw new Error(`the location record of ${appName} could not be read`); + return valid; +} + +/** + * Decides an app held in globalState.unconfirmedApps once its location record can + * be read, reading it every BOOT_STEP_RETRY_MS until then. A valid record releases + * the app and starts the components boot starts; any other answer removes it, the + * app held until the removal has finished or holds it as departing. + * @param {string} appName + * @param {string[]} componentsToStart + * @param {string} localSocketAddr + * @returns {Promise} + */ +async function confirmHeldApp(appName, componentsToStart, localSocketAddr) { + await serviceHelper.delay(BOOT_STEP_RETRY_MS); + const valid = await untilSucceeds(`Location check of ${appName}`, () => readLocation(appName, localSocketAddr)); + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.BOOT_BEFORE_LOCATION_DECISION, appName); + fluxEventBus.publish('boot:appConfirmed', { name: appName, valid }); + if (valid) { + log.info(`appStartupManager - App ${appName}: its location record is valid, starting it`); + globalState.unconfirmedApps.delete(appName); + startAtBoot(componentsToStart); + return; + } + log.warn(`appStartupManager - App ${appName}: no valid location record for this node (${localSocketAddr}), removing locally`); + try { + await removeAppBootMayNotStart(appName); + } finally { + globalState.unconfirmedApps.delete(appName); + } +} /** * Reconcile local app state against the network on boot. For each stopped * container, checks whether this node still has a valid location record. * - Valid location, non-g components: started. * - Valid location, g: components: left stopped (managed by masterSlaveApps). * - Expired/missing location: app removed locally (node was reassigned). + * - Unreadable location: app held stopped until the record can be read, then + * decided as above (confirmHeldApp). * @returns {Promise} Results of the reconciliation */ async function reconcileAppsOnBoot() { @@ -145,6 +205,7 @@ async function reconcileAppsOnBoot() { appsSkippedGMode: [], appsSkippedNoSpec: [], appsRemoved: [], + appsUnconfirmed: [], // location unreadable: held stopped until it can be read appsFailed: [], }; @@ -238,14 +299,23 @@ async function reconcileAppsOnBoot() { if (localSocketAddr) { // eslint-disable-next-line no-await-in-loop const hasValidLocation = await appHasValidLocationOnNode(appName, localSocketAddr); - if (hasValidLocation !== true) { - // An app whose location this node holds no longer, or cannot read, may have been - // placed elsewhere while this node was down: it is removed, never started. One it - // cannot confirm is announced as removed; an expired one the network has dropped. - const unconfirmed = hasValidLocation === null; - log.warn(`appStartupManager - App ${appName}: ${unconfirmed ? 'its location record could not be read' : `no valid location record for this node (${localSocketAddr})`}, removing locally`); + if (hasValidLocation === null) { + // Whether the network placed it elsewhere while this node was down is not + // known, so it is neither started nor removed until it is. + log.warn(`appStartupManager - App ${appName}: its location record could not be read, held stopped until it can be`); + globalState.unconfirmedApps.add(appName); + fluxEventBus.publish('boot:appUnconfirmed', { name: appName }); + confirmHeldApp(appName, componentsToStart, localSocketAddr) + .catch((error) => log.error(`appStartupManager - App ${appName}: the location check stopped: ${error.message}`)); + results.appsUnconfirmed.push(appName); + // eslint-disable-next-line no-continue + continue; + } + if (!hasValidLocation) { + // placed elsewhere while this node was down: removed, never started + log.warn(`appStartupManager - App ${appName}: no valid location record for this node (${localSocketAddr}), removing locally`); // eslint-disable-next-line no-await-in-loop - const removed = await removeAppBootMayNotStart(appName, unconfirmed); + const removed = await removeAppBootMayNotStart(appName); if (removed) results.appsRemoved.push(appName); else results.appsFailed.push({ app: appName, error: 'not removed; held stopped until it is' }); // eslint-disable-next-line no-await-in-loop @@ -263,13 +333,7 @@ async function reconcileAppsOnBoot() { log.info(`appStartupManager - App ${appName} has valid location, starting app`); } - // hand each component to the reconciler; it actuates once the boot gate - // opens (daemon/DB ready) and applies desired-state + restart policy - // eslint-disable-next-line no-restricted-syntax - for (const identifier of componentsToStart) { - appReconciler.enqueue(identifier); - log.info(`appStartupManager - Queued ${identifier} for reconcile`); - } + startAtBoot(componentsToStart); if (isPartial) { results.appsPartiallyStarted.push(appName); @@ -284,6 +348,7 @@ async function reconcileAppsOnBoot() { + `Apps started: ${results.appsStarted.length}, ` + `Apps partially started (mixed g:): ${results.appsPartiallyStarted.length}, ` + `Apps removed (expired location): ${results.appsRemoved.length}, ` + + `Apps held (location unreadable): ${results.appsUnconfirmed.length}, ` + `Apps skipped (g: mode): ${results.appsSkippedGMode.length}, ` + `Apps skipped (no spec): ${results.appsSkippedNoSpec.length}, ` + `Apps failed: ${results.appsFailed.length}`, diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index bd988ef6fc..de5ecc219c 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -188,10 +188,13 @@ async function checkAndNotifyPeersOfRunningApps() { // installed until the removal finishes, and naming it here would re-create the // location row the removal just cleared. An app this node is only testing is // excluded too: its row is thrown away at the end of the test, and the teardown - // that throws it away tells the network nothing. + // that throws it away tells the network nothing. So is an app boot has not yet + // confirmed this node still holds: naming it would renew a location record that + // may have expired, and the boot check would then read it as valid. const applicationsToBroadcast = appsInstalled.filter( (application) => !globalState.departingApps.has(application.name) - && !globalState.testInstallingApps.has(application.name), + && !globalState.testInstallingApps.has(application.name) + && !globalState.unconfirmedApps.has(application.name), ); const apps = []; const db = dbHelper.databaseConnection(); diff --git a/ZelBack/src/services/appMonitoring/appReconciler.js b/ZelBack/src/services/appMonitoring/appReconciler.js index ac409bed64..40728bd72d 100644 --- a/ZelBack/src/services/appMonitoring/appReconciler.js +++ b/ZelBack/src/services/appMonitoring/appReconciler.js @@ -377,6 +377,8 @@ function isManagedElsewhere(identifier) { if (globalState.stoppingContainers.has(dockerService.getAppIdentifier(identifier))) return true; // an app being removed, including one whose removal is still cleaning up if (globalState.departingApps.has(mainAppName)) return true; + // an app boot has not yet confirmed this node still holds + if (globalState.unconfirmedApps.has(mainAppName)) return true; return false; } @@ -745,6 +747,7 @@ async function healDetachedNetwork(identifier, mainAppName, spec) { async function reconcile(rawIdentifier) { const identifier = canonical(rawIdentifier); if (isManagedElsewhere(identifier)) { + fluxEventBus.count('reconciler:deferred', identifier, 'managedElsewhere'); scheduleRetry(identifier, MANAGED_RETRY_MS); return; } diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index d82b3d9f7b..fe2ec33917 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -87,6 +87,9 @@ const Checkpoint = Object.freeze({ // This node has recorded a move of its own address and has not yet acted on // it. Keyed by the address it moved to. OWN_ADDRESS_BEFORE_RECONCILE: 'ownAddress:beforeReconcile', + // Boot has read the location record of an app it held unconfirmed, and has not + // yet released the app or acted on the answer. Keyed by app name. + BOOT_BEFORE_LOCATION_DECISION: 'boot:beforeLocationDecision', }); const CHECKPOINT_NAMES = new Set(Object.values(Checkpoint)); // Holds a key for every key of its checkpoint. diff --git a/ZelBack/src/services/utils/globalState.js b/ZelBack/src/services/utils/globalState.js index a73bc2d960..ab86792bb5 100644 --- a/ZelBack/src/services/utils/globalState.js +++ b/ZelBack/src/services/utils/globalState.js @@ -124,6 +124,14 @@ const departingApps = { }, }; +// Apps boot found stopped and whose location records it could not read, so it cannot +// tell whether this node still holds them or the network placed them elsewhere while +// it was down. Until boot reads the records, an app here is not started, elected, +// announced, backed up, restored or torn down to be rebuilt. +// +// In-memory deliberately: a restart runs the boot check again. +const unconfirmedApps = new Set(); + // Containers intentionally stopped by FluxOS — crash recovery skips die events for these const stoppingContainers = new Set(); @@ -363,6 +371,7 @@ module.exports = { get runningAppsCache() { return runningAppsCache; }, get departingApps() { return departingApps; }, get testInstallingApps() { return testInstallingApps; }, + get unconfirmedApps() { return unconfirmedApps; }, get announceCycle() { return announceCycle; }, get stoppingContainers() { return stoppingContainers; }, get fluxRemovedContainers() { return fluxRemovedContainers; }, diff --git a/test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js similarity index 100% rename from test-infra/runner/tests/3308-boot-removes-an-app-it-cannot-confirm.js rename to test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 0ae676dd96..755a42459c 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -1010,6 +1010,54 @@ describe('advancedWorkflows tests', () => { } }); + // Electing an app sets its folder's type - sending on a primary, receive-only on + // a standby - and the reconciler starts none of its components while it is held. + // Each hold is checked beside the same app unheld, which the election does decide. + describe('an app the node holds back from election', () => { + const appName = 'testapp'; + const runPass = async () => { + const installedApps = sinon.stub().resolves({ + status: 'success', + data: [{ name: appName, version: 3, containerData: 'g:/data' }], + }); + const listRunningApps = sinon.stub().resolves({ status: 'success', data: [] }); + serviceHelperStub.resolves(fdmNoPrimary()); + await advancedWorkflows.masterSlaveApps(globalState, installedApps, listRunningApps, https); + }; + + it('decides the app when nothing holds it', async () => { + const count = sinon.stub(fluxEventBus, 'count'); + await runPass(); + sinon.assert.calledWith(count, 'masterSlave:decision', sinon.match.string, 'evaluated'); + }); + + it('leaves an app whose removal is pending undecided', async () => { + const count = sinon.stub(fluxEventBus, 'count'); + globalState.departingApps.enter(appName); + try { + await runPass(); + } finally { + globalState.departingApps.leave(appName); + } + sinon.assert.calledWith(count, 'masterSlave:decision', appName, 'skippedDeparting'); + sinon.assert.neverCalledWith(count, 'masterSlave:decision', sinon.match.string, 'evaluated'); + expect(serviceHelperStub.called, 'FDM was asked about an app the election leaves alone').to.be.false; + }); + + it('leaves an app boot has not yet confirmed it holds undecided', async () => { + const count = sinon.stub(fluxEventBus, 'count'); + globalState.unconfirmedApps.add(appName); + try { + await runPass(); + } finally { + globalState.unconfirmedApps.delete(appName); + } + sinon.assert.calledWith(count, 'masterSlave:decision', appName, 'skippedUnconfirmed'); + sinon.assert.neverCalledWith(count, 'masterSlave:decision', sinon.match.string, 'evaluated'); + expect(serviceHelperStub.called, 'FDM was asked about an app the election leaves alone').to.be.false; + }); + }); + // Shared fixture for the recovery tests below: a v3 g: app with this node in // the location list. `peers` are the other nodes, in election order. // `selfRunningSince` places this node in the election order; `receiveOnlyCache` @@ -5619,6 +5667,42 @@ describe('advancedWorkflows tests', () => { expect(globalState.backupInProgress).to.not.include(appname); }); + // A backup ends by starting the app, so it is refused for an app the node is + // removing or has not yet confirmed it still holds. + it('refuses an app whose removal is pending', async () => { + const res = makeRes(); + globalState.departingApps.enter(appname); + let result; + try { + result = await advancedWorkflows.appendBackupTask(backupReq(), res); + } finally { + globalState.departingApps.leave(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is being removed from this node'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.backupInProgress).to.not.include(appname); + }); + + it('refuses an app boot has not yet confirmed it holds', async () => { + const res = makeRes(); + globalState.unconfirmedApps.add(appname); + let result; + try { + result = await advancedWorkflows.appendBackupTask(backupReq(), res); + } finally { + globalState.unconfirmedApps.delete(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is held until this node confirms it still holds it'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.backupInProgress).to.not.include(appname); + }); + it('refuses cleanly when the app has no specification, not with a TypeError', async () => { // The restore guards this; the backup read the spec and handed it straight // to syncedComponentsOfApp, which threw on null deep in the flow instead of @@ -6003,6 +6087,42 @@ describe('advancedWorkflows tests', () => { expect(globalState.restoreInProgress).to.not.include(appname); }); + // A restore ends by starting the app, so it is refused for an app the node + // is removing or has not yet confirmed it still holds. + it('refuses an app whose removal is pending', async () => { + const res = makeRes(); + globalState.departingApps.enter(appname); + let result; + try { + result = await advancedWorkflows.appendRestoreTask(restoreReq(), res); + } finally { + globalState.departingApps.leave(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is being removed from this node'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.restoreInProgress).to.not.include(appname); + }); + + it('refuses an app boot has not yet confirmed it holds', async () => { + const res = makeRes(); + globalState.unconfirmedApps.add(appname); + let result; + try { + result = await advancedWorkflows.appendRestoreTask(restoreReq(), res); + } finally { + globalState.unconfirmedApps.delete(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is held until this node confirms it still holds it'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.restoreInProgress).to.not.include(appname); + }); + it('refuses a component the app does not have', async () => { const req = restoreReq(); req.body.restore = [{ component: 'palworld; rm -rf /', restore: true, url: 'https://example.invalid/a.tar.gz' }]; @@ -8645,6 +8765,17 @@ describe('a redeploy asks what it may rebuild before it takes anything down', () expect(res.write.firstCall.args[0]).to.match(/policy/i); }); + // A rebuilt app starts, and boot has yet to decide whether this one may. + it('declines while boot has not confirmed the node still holds the app, and leaves it where it is', async () => { + globalState.unconfirmedApps.add('plainapp'); + + const proceed = await advancedWorkflows.mayTearDownToRebuild({ name: 'plainapp', nodes: [] }, res); + + expect(proceed, 'the teardown must not start').to.be.false; + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.write.firstCall.args[0]).to.include('redeploy deferred'); + }); + it('hands back an app whose spec names other nodes, and tells the network', async () => { const proceed = await advancedWorkflows.mayTearDownToRebuild(pinnedSpec(['10.0.0.9:16127']), res); diff --git a/tests/unit/appReconciler.test.js b/tests/unit/appReconciler.test.js index 5e4112cd83..02a5676fe7 100644 --- a/tests/unit/appReconciler.test.js +++ b/tests/unit/appReconciler.test.js @@ -458,6 +458,16 @@ describe('appReconciler tests', () => { stubs.globalState.departingApps.leave('App'); }); + it('leaves a component alone while boot has not confirmed its app is still held here', async () => { + stubs.globalState.unconfirmedApps.add('App'); + + await appReconciler.reconcile('www_App'); + + expect(stubs.dockerService.dockerContainerInspect.called, 'an unconfirmed app was reconciled').to.be.false; + expect(stubs.dockerService.appDockerStart.called).to.be.false; + stubs.globalState.unconfirmedApps.delete('App'); + }); + it('starts nothing when the stop lock cannot be read, and fails the pass for a retry', async () => { stubs.appsRuntimeState.operatorStopState.rejects(new Error('connection timed out')); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 5f74bb0d88..df06284cb8 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -100,6 +100,7 @@ describe('appStartupManager tests', () => { '../utils/globalState': globalStateStub, // runs the step again at once until it succeeds, as the real one does after a wait '../utils/untilSucceeds': { + BOOT_STEP_RETRY_MS: 15000, untilSucceeds: async (what, step) => { for (;;) { try { @@ -342,20 +343,113 @@ describe('appStartupManager tests', () => { sinon.assert.calledOnceWithExactly(appUninstallerStub.finishRemoval, 'AppA', true, false); }); - it('removes an app whose location record cannot be read, and tells the network', async () => { - // Its location may have been given to another node while this one was down, so - // it is never started unconfirmed. - dockerServiceStub.dockerListContainers.resolves([ - { Names: ['/fluxAppA'], State: 'exited' }, - ]); - dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); - dbHelperStub.findInDatabase.onSecondCall().rejects(new Error('DB error')); + describe('an app whose location record cannot be read', () => { + const futureExpiry = () => new Date(Date.now() + (300 * 1000)); + // the held app's check runs in the background, on stubs that answer at once + const settle = () => new Promise((resolve) => { setImmediate(resolve); }); - const results = await appStartupManager.reconcileAppsOnBoot(); + beforeEach(() => { + dockerServiceStub.dockerListContainers.resolves([ + { Names: ['/fluxAppA'], State: 'exited' }, + ]); + dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); + dbHelperStub.findInDatabase.onSecondCall().rejects(new Error('DB error')); + }); - sinon.assert.calledOnceWithExactly(appUninstallerStub.removeAppLocally, 'AppA', null, true, true, true); - expect(results.appsRemoved).to.deep.equal(['AppA']); - expect(appReconcilerStub.enqueue.calledWith('AppA')).to.equal(false); + it('is held, neither started nor removed', async () => { + // the record stays unreadable for as long as this test looks + dbHelperStub.findInDatabase.onThirdCall().returns(new Promise(() => {})); + + const results = await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + expect(results.appsUnconfirmed).to.deep.equal(['AppA']); + expect(globalStateStub.unconfirmedApps.has('AppA'), 'the app was not held').to.equal(true); + expect(appUninstallerStub.removeAppLocally.called, 'an app whose location is unknown was removed').to.equal(false); + expect(appUninstallerStub.finishRemoval.called, 'an app whose location is unknown was handed to be removed').to.equal(false); + expect(appReconcilerStub.enqueue.called, 'an app whose location is unknown was started').to.equal(false); + }); + + it('is started, and released, once its record reads valid', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([{ expireAt: futureExpiry() }]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appReconcilerStub.enqueue, 'AppA'); + expect(globalStateStub.unconfirmedApps.has('AppA'), 'a confirmed app is still held').to.equal(false); + expect(appUninstallerStub.removeAppLocally.called).to.equal(false); + }); + + it('has its record read again until it answers', async () => { + dbHelperStub.findInDatabase.onThirdCall().rejects(new Error('DB error')); + dbHelperStub.findInDatabase.onCall(3).rejects(new Error('DB error')); + dbHelperStub.findInDatabase.onCall(4).resolves([{ expireAt: futureExpiry() }]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + expect(dbHelperStub.findInDatabase.callCount, 'the record was not read until it answered').to.equal(5); + sinon.assert.calledOnceWithExactly(appReconcilerStub.enqueue, 'AppA'); + }); + + it('is removed once its record reads expired, telling the network nothing', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appUninstallerStub.removeAppLocally, 'AppA', null, true, true, false); + expect(appReconcilerStub.enqueue.called, 'an app whose location expired was started').to.equal(false); + expect(globalStateStub.unconfirmedApps.has('AppA')).to.equal(false); + }); + + it('stays held until its removal has run', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + let finishRemoval; + appUninstallerStub.removeAppLocally.returns(new Promise((resolve) => { finishRemoval = resolve; })); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + expect(globalStateStub.unconfirmedApps.has('AppA'), 'released while its removal was running').to.equal(true); + + finishRemoval('removed'); + await settle(); + expect(globalStateStub.unconfirmedApps.has('AppA')).to.equal(false); + }); + + it('is handed to be finished when its removal does not finish', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + appUninstallerStub.removeAppLocally.resolves('busy'); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appUninstallerStub.finishRemoval, 'AppA', true, false); + expect(appReconcilerStub.enqueue.called).to.equal(false); + }); + + it('has only the components boot starts started, when its g: components are left to the election', async () => { + dockerServiceStub.dockerListContainers.resolves([ + { Names: ['/fluxweb_MixedApp'], State: 'exited' }, + { Names: ['/fluxdb_MixedApp'], State: 'exited' }, + ]); + dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'MixedApp' }]); + registryManagerStub.getApplicationGlobalSpecifications.withArgs('MixedApp').resolves({ + version: 8, + name: 'MixedApp', + compose: [ + { name: 'web', containerData: '' }, + { name: 'db', containerData: 'g:/data' }, + ], + }); + dbHelperStub.findInDatabase.onThirdCall().resolves([{ expireAt: futureExpiry() }]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appReconcilerStub.enqueue, 'web_MixedApp'); + }); }); it('should check location after g: syncthing check', async () => { diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index 45bd5a965b..254d379192 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -17,6 +17,7 @@ describe('peerNotification tests', () => { // same object. let departingApps; let testInstallingApps; + let unconfirmedApps; let announceCycle; let findOneInDatabaseStub; @@ -109,6 +110,7 @@ describe('peerNotification tests', () => { '../utils/globalState': { departingApps, testInstallingApps, + unconfirmedApps, announceCycle, runningAppsCache: new Set(), }, @@ -141,7 +143,9 @@ describe('peerNotification tests', () => { // this suite's alone - evicting the entry instead would hand another object // to every module loaded after it, and globalState is a singleton whose // flags decide whether an operation may start at all. - ({ departingApps, testInstallingApps, announceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {})); + ({ + departingApps, testInstallingApps, unconfirmedApps, announceCycle, + } = proxyquire('../../ZelBack/src/services/utils/globalState', {})); peerNotification = loadPeerNotification(); }); @@ -240,6 +244,40 @@ describe('peerNotification tests', () => { expect(message.apps.map((a) => a.name)).to.deep.equal(['app1']); }); + // Boot holds an app whose location record it could not read. Naming it would + // renew a record that may have expired, and the boot check would then read the + // renewed record as valid and start the app beside the one placed elsewhere. + it('does not announce an app boot has not yet confirmed it holds', async () => { + installedAppsStub.resolves({ + status: 'success', + data: [ + { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + ], + }); + unconfirmedApps.add('app2'); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect( + message.apps.map((a) => a.name), + 'the unconfirmed app is excluded and the rest still announced', + ).to.deep.equal(['app1']); + }); + + it('announces the app again once boot has confirmed it', async () => { + unconfirmedApps.add('app1'); + await peerNotification.checkAndNotifyPeersOfRunningApps(); + expect(storeAppRunningMessageStub.called, 'announced while unconfirmed').to.be.false; + + unconfirmedApps.delete('app1'); + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps.map((a) => a.name)).to.deep.equal(['app1']); + }); + it('announces nothing when every installed app is departing', async () => { departingApps.enter('app1'); From c796b51503334d813c606faca13f876783c9e06f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:05:37 +0100 Subject: [PATCH 171/369] test(harness): 3308, boot holds an app whose location it cannot read and decides it once it can Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/db-client.js | 21 + test-infra/runner/framework/node-client.js | 4 + ...308-boot-holds-an-app-it-cannot-confirm.js | 414 ++++++++++++++---- 3 files changed, 361 insertions(+), 78 deletions(-) diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index 0c0b6bce67..ff3ebb21de 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -113,6 +113,27 @@ export function dbClient(nodeNum) { // This node's view of where an app runs (the rows fluxapprunning messages // upsert and the empty-broadcast bug used to wipe). + // A location's expiry on this node: its rows for the app at that address are + // gone, as the TTL index removes them. Resolves to how many rows went. + async expireAppLocation(appName, ip) { + const globalDb = await db('appsGlobal'); + const host = String(ip).replace(/:\d+$/, '').replace(/\./g, '\\.'); + const { deletedCount } = await globalDb.collection('zelappslocation') + .deleteMany({ name: appName, ip: { $regex: `^${host}(:\\d+)?$` } }); + return deletedCount; + }, + + // This node's rows for the app at that address stay unexpired for `ms` from now, + // as a production record does across a restart the harness's compressed record + // would not outlive. Resolves to how many rows were found. + async extendAppLocation(appName, ip, ms) { + const globalDb = await db('appsGlobal'); + const host = String(ip).replace(/:\d+$/, '').replace(/\./g, '\\.'); + const { matchedCount } = await globalDb.collection('zelappslocation') + .updateMany({ name: appName, ip: { $regex: `^${host}(:\\d+)?$` } }, { $set: { expireAt: new Date(Date.now() + ms) } }); + return matchedCount; + }, + async getAppLocations(appName) { const globalDb = await db('appsGlobal'); return globalDb.collection('zelappslocation').find({ name: appName }).toArray(); diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 11df4b747d..ea46f74f37 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -195,6 +195,10 @@ export function nodeClient(nodeNum) { 'masterSlave:operatorStartSettled', 'stream:gap', 'boot:settled', + // an app boot could not confirm this node still holds, held stopped; and the + // answer once its location record could be read + 'boot:appUnconfirmed', + 'boot:appConfirmed', 'confirmation:changed', 'daemon:polled', 'daemon:recovered', diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index f92b6732f9..516e628620 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -1,118 +1,376 @@ +// weight: heavy /* - * At boot a node reads the network's location record of each app it holds: one whose - * record has expired was placed elsewhere while the node was down. When the record - * cannot be read, the node cannot tell, so it never starts the app: it removes it and - * tells the network. + * At boot a node reads its location record of each app whose containers are stopped, + * which is every app a machine reboot leaves. One whose record has expired was placed + * elsewhere while the node was down and is removed; one whose record is valid is + * started. When the record cannot be read the node cannot tell, so the app is held: + * not started, not elected, not announced. Boot reads the record again until it + * answers, then starts or removes the app by the answer. * - * The check runs for apps whose containers are stopped, which is what a machine reboot - * leaves. Here FluxOS is killed and held down, the app's container is stopped, and the - * node's reads of the location collection are refused before FluxOS comes back. + * Here FluxOS is killed and held down, the apps' containers are stopped, and the + * node's reads of the location collection are refused before FluxOS comes back. The + * holder's records are made to outlive the restart, as a production record does and + * the harness's compressed one would not; a record meant to have expired is deleted. + * + * The mixed app pairs an active-standby component, which only the election starts, + * with a plain one boot starts. Its decision is held at a checkpoint once the + * database answers, so the announcement and the election that run then can be + * watched leaving it alone. The plain app is decided as soon as the database + * answers: it is the canary that the announcement and the decisions run at all. + * + * Every app asks for as many instances as the fleet has nodes and is on every node + * before the holder goes down, so none is placed anew while the holder is away. A + * standby may take over the mixed app's active-standby component in that time. */ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { pushImage } from '../framework/registry-helper.js'; +import { pushImage, pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { bootAndPeer, installOnNodes, installedInstanceIndices } from '../framework/reconciler-suite.js'; import { - crashFluxos, getAppContainerId, isAppContainerRunning, releaseFluxos, stopAppContainer, + crashFluxos, listAppContainers, releaseFluxos, stopAppContainer, } from '../framework/container.js'; import { waitFor } from '../framework/wait.js'; import { failCommands } from '../framework/mongo-control.js'; +import { dbClient } from '../framework/db-client.js'; +import { isDaemonUp } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const HOLDER = 0; +const NODES = 3; +const APP_UID = 1000; +const BEFORE_DECISION = 'boot:beforeLocationDecision'; +// Longer than anything below, and far shorter than a production record's TTL. +const RECORD_OUTLIVES_MS = 60 * 60 * 1000; function step(what) { console.log(`# 3308 ${new Date().toISOString()} ${what}`); } -describe('boot removes an app whose location it cannot confirm', function () { +function plainComponent(appName) { + return { + name: appName, + description: 'boot location test container', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +describe('boot holds an app whose location it cannot read', function () { let env; dumpLogsOnFailure(() => env); - const appName = `e2ebootunread${Date.now()}`; + const stamp = Date.now(); + // decided at once when the database answers + const plainApp = `e2ebootplain${stamp}`; + // decided once the checkpoint is released + const mixedApp = `e2ebootmixed${stamp}`; + // its record is gone + const lapsedApp = `e2ebootlapsed${stamp}`; + const db = `db_${mixedApp}`; + const web = `web_${mixedApp}`; + const plain = `${plainApp}_${plainApp}`; + const lapsed = `${lapsedApp}_${lapsedApp}`; let holder; - let release; - let anchors; - let failedReads; - let stoppedAtBoot; - let heldAfterBoot; + let holderDb; + const activeStandby = watchGComponents(() => env, { + appName: mixedApp, components: ['db'], holders: [...Array(NODES).keys()], + }); - before(async function () { - this.timeout(600000); - env = await createTestEnv({ hookCtx: this, nodes: 5, tickerAutostart: false }); - await bootAndPeer(env); - holder = env.clients[HOLDER]; + const isUp = async (identifier) => (await listAppContainers(holder.container)) + .some((c) => c.name === `flux${identifier}` && c.status?.startsWith('Up')); + const installedNames = async () => ((await holder.getInstalledApps()).data ?? []).map((a) => a.name); + const heardRemoved = (appName, afterIds) => env.clients.some((client, i) => i !== HOLDER && client + && client.getEventBuffer().some((e) => e.id > afterIds[i] && e.event === 'network:appremoved' + && e.data?.name === appName && e.data?.ip?.startsWith(holder.ip))); + const heardRunning = (appName, afterIds) => env.clients.some((client, i) => i !== HOLDER && client + && client.getEventBuffer().some((e) => e.id > afterIds[i] && e.event === 'network:apprunning' + && e.data?.ip?.startsWith(holder.ip) && (e.data?.apps ?? []).some((a) => a.name === appName))); + // What the reconciler did with a component it was asked about after `afterId`: + // deferred it or started it, resolved once it has done one of the two. + const reconcilerAnswer = async (identifier, afterId) => { + let answer = null; + await waitFor(async () => { + const started = holder.getEventBuffer().some((e) => e.id > afterId && e.event === 'reconciler:actuated' + && e.data?.identifier === identifier && e.data?.action === 'started'); + if (started) { + answer = 'started'; + return true; + } + if (await holder.getDecisionCount('reconciler:deferred', identifier, 'managedElsewhere') > 0) { + answer = 'deferred'; + return true; + } + return false; + }, { timeout: 120000, interval: 1000, label: `the reconciler answered for ${identifier}` }); + return answer; + }; + const cycles = async () => (await holder.getTestCounters())?.['masterSlave:cycles'] ?? 0; - await pushImage(appName, 'v1'); - const app = await buildSeedableApp({ - env, - name: appName, - compose: [{ - name: appName, - description: 'boot location test container', - repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, - ports: [], - domains: [''], - environmentParameters: [], - commands: [], - containerPorts: [80], - containerData: '/tmp', - cpu: 0.1, - ram: 100, - hdd: 1, - repoauth: '', - }], - }); - await installOnNodes(env, app, [HOLDER]); - await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)), { - timeout: 120000, interval: 2000, label: 'the app runs before the restart', - }); - - step('taking the node down, its app container stopped and its location reads refused'); + // FluxOS killed and held, the given containers stopped, the holder's records of + // `keep` made to outlive the restart and those of `drop` deleted, and its reads of + // the location collection refused from before FluxOS comes back. + async function rebootUnreadable(identifiers, { keep, drop }) { await crashFluxos(holder.container, { hold: true }); - const stopped = await stopAppContainer(holder.container, appName, appName); - if (stopped.exitCode !== 0) throw new Error(`could not stop the app container: ${stopped.output}`); - stoppedAtBoot = !(await isAppContainerRunning(holder.container, appName)); - anchors = env.clients.map((client) => client?.getLastEventId()); - release = await failCommands(HOLDER + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); + for (const identifier of identifiers) { + const [component, appName] = identifier.split('_'); + // eslint-disable-next-line no-await-in-loop + const stopped = await stopAppContainer(holder.container, appName, component); + if (stopped.exitCode !== 0) throw new Error(`could not stop ${identifier}: ${stopped.output}`); + } + for (const appName of keep) { + // eslint-disable-next-line no-await-in-loop + const kept = await holderDb.extendAppLocation(appName, holder.ip, RECORD_OUTLIVES_MS); + if (kept < 1) throw new Error(`the holder has no location record of ${appName} to keep`); + } + for (const appName of drop) { + // eslint-disable-next-line no-await-in-loop + const gone = await holderDb.expireAppLocation(appName, holder.ip); + if (gone < 1) throw new Error(`the holder has no location record of ${appName} to expire`); + } + const anchors = env.clients.map((client) => client?.getLastEventId()); + const release = await failCommands(HOLDER + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); await releaseFluxos(holder.container, { readyTimeoutMs: 180000 }); - await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); - // read at boot's end: the node's own spawner may place the app again later - heldAfterBoot = (await holder.getInstalledApps()).data?.some((a) => a.name === appName) ?? null; - failedReads = await release(); - release = null; - step('booted'); + return { anchors, release }; + } + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + holder = env.clients[HOLDER]; + holderDb = dbClient(HOLDER + 1); + await Promise.all(env.clients.map((client, i) => waitFor(() => isDaemonUp(client), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(mixedApp, 'v1', 'bootmixed', { user: APP_UID }); + await pushImage(plainApp, 'v1'); + await pushImage(lapsedApp, 'v1'); + const apps = await Promise.all([ + buildSeedableApp({ + env, + name: mixedApp, + instances: NODES, + compose: [gComponent(mixedApp, 'db'), { ...gComponent(mixedApp, 'web'), containerData: '/appdata' }], + }), + buildSeedableApp({ env, name: plainApp, instances: NODES, compose: [plainComponent(plainApp)] }), + buildSeedableApp({ env, name: lapsedApp, instances: NODES, compose: [plainComponent(lapsedApp)] }), + ]); + for (const app of apps) { + // eslint-disable-next-line no-await-in-loop + await installOnNodes(env, app, [HOLDER]); + } + await waitFor(async () => (await Promise.all([db, web, plain, lapsed].map(isUp))).every(Boolean), { + timeout: 300000, interval: 3000, label: 'every app runs on the holder, the mixed app\'s db as its primary', + }); + await waitFor(async () => (await Promise.all([mixedApp, plainApp, lapsedApp].map((appName) => installedInstanceIndices(env, appName)))) + .every((holders) => holders.length === NODES), { + timeout: 300000, interval: 5000, label: 'every app is on every node', + }); }); after(async function () { this.timeout(60000); - await release?.().catch(() => {}); await env?.teardown(); }); - it('boots with the app\'s container stopped', function () { - expect(stoppedAtBoot, 'the app\'s container was running when FluxOS came back, so boot had nothing to check').to.equal(true); - }); + describe('a record that reads valid once it can be read', function () { + let anchors; + let release; + let failedReads; + let stoppedAtBoot; + let sweptAfterBoot; + let webWhileUnreadable; + let plainWhileUnreadable; + let upWhileUnreadable; + let installedWhileUnreadable; + let announcement; + let electionCycles; + let evaluatedWhileHeld; + let skippedWhileHeld; + let upWhileHeld; - it('had its reads of the location records refused by the database', function () { - expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); - }); + before(async function () { + this.timeout(900000); + step('taking the node down, its containers stopped and its location reads refused'); + ({ anchors, release } = await rebootUnreadable([db, web, plain], { keep: [mixedApp, plainApp], drop: [] })); + // Held before boot's first read, which fails: the decision it holds comes at + // least one retry interval after that. + await holder.holdCheckpoint(BEFORE_DECISION, mixedApp); + const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + await Promise.all([mixedApp, plainApp].map((appName) => holder.waitForEvent( + 'boot:appUnconfirmed', (d) => d.name === appName, 60000, { afterId: anchors[HOLDER] }, + ))); + stoppedAtBoot = !(await Promise.all([db, web, plain].map(isUp))).some(Boolean); + + step('the location records unreadable: the reconciler is asked about every component'); + sweptAfterBoot = Boolean(await holder.waitForEvent('reconciler:swept', () => true, 120000, { afterId: settled.id }) + .catch(() => null)); + [webWhileUnreadable, plainWhileUnreadable] = await Promise.all([ + reconcilerAnswer(web, settled.id), reconcilerAnswer(plain, settled.id), + ]); + upWhileUnreadable = (await Promise.all([db, web, plain].map(isUp))).some(Boolean); + installedWhileUnreadable = await installedNames(); + + step('the database answers: the plain app is decided, the mixed app held at its decision'); + const answeredFrom = holder.getLastEventId(); + const evaluatedBefore = await holder.getDecisionCount('masterSlave:decision', db, 'evaluated'); + failedReads = await release(); + release = null; + await holder.waitForEvent('checkpoint:held', (d) => d.name === BEFORE_DECISION && d.key === mixedApp, 120000, { afterId: answeredFrom }); + await holder.waitForEvent('boot:appConfirmed', (d) => d.name === plainApp && d.valid === true, 120000, { afterId: answeredFrom }); + announcement = (await holder.waitForEvent( + 'app:running', (d) => (d.apps ?? []).some((a) => a.name === plainApp), 120000, { afterId: answeredFrom }, + ).catch(() => null))?.data ?? null; + const cyclesFrom = await cycles(); + await waitFor(async () => (await cycles()) >= cyclesFrom + 2, { + timeout: 180000, interval: 2000, label: 'the election ran two passes while the mixed app was held', + }).catch(() => {}); + electionCycles = (await cycles()) - cyclesFrom; + evaluatedWhileHeld = (await holder.getDecisionCount('masterSlave:decision', db, 'evaluated')) - evaluatedBefore; + skippedWhileHeld = await holder.getDecisionCount('masterSlave:decision', mixedApp, 'skippedUnconfirmed'); + upWhileHeld = (await Promise.all([db, web].map(isUp))).some(Boolean); + + step('the mixed app\'s decision released'); + await holder.releaseCheckpoint(BEFORE_DECISION, mixedApp); + await holder.waitForEvent('boot:appConfirmed', (d) => d.name === mixedApp && d.valid === true, 60000, { afterId: answeredFrom }); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await holder?.releaseAllCheckpoints().catch(() => {}); + }); + + it('boots with the apps\' containers stopped', function () { + expect(stoppedAtBoot, 'a container was running when FluxOS came back, so boot had nothing to check').to.equal(true); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('keeps an app it cannot confirm installed', function () { + expect(installedWhileUnreadable, 'boot removed an app whose location it could not read').to.include.members([mixedApp, plainApp]); + }); + + it('starts no component of an app it cannot confirm, when the reconciler is asked to', function () { + expect(sweptAfterBoot, 'no reconciler sweep ran while the records were unreadable, so nothing below was tested').to.equal(true); + expect(webWhileUnreadable, 'the reconciler started a component of an app boot could not confirm').to.equal('deferred'); + expect(plainWhileUnreadable, 'the reconciler started an app boot could not confirm').to.equal('deferred'); + expect(upWhileUnreadable, 'a component of an app boot could not confirm is running').to.equal(false); + }); + + it('leaves an app it has not confirmed out of the announcement once the database answers', function () { + expect(announcement, 'the node announced nothing once the database answered, so nothing below was tested').to.not.equal(null); + expect(announcement.apps.map((a) => a.name), 'the announcement named an app boot had not confirmed').to.not.include(mixedApp); + }); - it('no longer holds the app once boot is over', function () { - expect(heldAfterBoot, 'boot left an app it could not confirm installed').to.equal(false); + it('leaves an app it has not confirmed out of the election', function () { + expect(electionCycles, 'the election ran no pass while the app was held, so nothing below was tested').to.be.at.least(2); + expect(evaluatedWhileHeld, 'the election decided an app boot had not confirmed').to.equal(0); + expect(skippedWhileHeld, 'the election did not leave the held app alone').to.be.at.least(1); + expect(upWhileHeld, 'a component of an app boot had not confirmed is running').to.equal(false); + }); + + it('starts the app once its record reads valid, its active-standby component on one node', async function () { + this.timeout(240000); + await waitFor(async () => (await Promise.all([web, plain].map(isUp))).every(Boolean), { + timeout: 200000, interval: 3000, label: 'the plain components run again', + }); + await waitFor(async () => Object.values(await activeStandby.oneHolderAtMost()).some((running) => running.includes('db')), { + timeout: 200000, interval: 3000, label: 'the mixed app\'s db runs on one node', + }); + expect(await installedNames()).to.include.members([mixedApp, plainApp]); + }); + + it('tells the network nothing of an app it kept', async function () { + this.timeout(120000); + await waitFor(() => heardRunning(plainApp, anchors), { + timeout: 100000, interval: 2000, label: 'a peer heard the holder announce once the database answered', + }); + expect(heardRemoved(mixedApp, anchors), 'a peer was told the holder gave up an app it kept').to.equal(false); + expect(heardRemoved(plainApp, anchors), 'a peer was told the holder gave up an app it kept').to.equal(false); + }); }); - it('tells the network it no longer holds the app', async function () { - this.timeout(90000); - const peers = env.clients.filter((client, i) => client && i !== HOLDER); - const heard = await Promise.any(peers.map((peer) => peer.waitForEvent( - 'network:appremoved', - (d) => d.name === appName && d.ip.startsWith(holder.ip), - 60000, - { afterId: anchors[env.clients.indexOf(peer)] }, - ))).catch(() => null); - expect(heard, 'no node was told the app had gone').to.not.equal(null); + describe('a record that reads expired once it can be read', function () { + let anchors; + let release; + let failedReads; + let sweptAfterBoot; + let lapsedWhileUnreadable; + let installedWhileUnreadable; + let removed; + let installedAfter; + + before(async function () { + this.timeout(900000); + step('taking the node down again, the lapsed app\'s record deleted'); + ({ anchors, release } = await rebootUnreadable([lapsed, plain], { keep: [plainApp], drop: [lapsedApp] })); + const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + await holder.waitForEvent('boot:appUnconfirmed', (d) => d.name === lapsedApp, 60000, { afterId: anchors[HOLDER] }); + sweptAfterBoot = Boolean(await holder.waitForEvent('reconciler:swept', () => true, 120000, { afterId: settled.id }) + .catch(() => null)); + lapsedWhileUnreadable = await reconcilerAnswer(lapsed, settled.id); + installedWhileUnreadable = await installedNames(); + + step('the database answers'); + const answeredFrom = holder.getLastEventId(); + failedReads = await release(); + release = null; + await holder.waitForEvent('boot:appConfirmed', (d) => d.name === lapsedApp && d.valid === false, 120000, { afterId: answeredFrom }); + removed = Boolean(await holder.waitForEvent('app:removed', (d) => d.name === lapsedApp, 120000, { afterId: answeredFrom }) + .catch(() => null)); + installedAfter = await installedNames(); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('holds the app, installed and stopped, while its record cannot be read', function () { + expect(sweptAfterBoot, 'no reconciler sweep ran while the records were unreadable, so nothing below was tested').to.equal(true); + expect(installedWhileUnreadable, 'boot removed an app whose location it could not read').to.include(lapsedApp); + expect(lapsedWhileUnreadable, 'the reconciler started an app boot could not confirm').to.equal('deferred'); + }); + + it('removes the app once its record reads expired', function () { + expect(removed, 'the app whose record expired was not removed').to.equal(true); + expect(installedAfter, 'the app whose record expired is still installed').to.not.include(lapsedApp); + }); + + it('tells the network nothing of the removal, the network having dropped the record', async function () { + this.timeout(120000); + await waitFor(() => heardRunning(plainApp, anchors), { + timeout: 100000, interval: 2000, label: 'a peer heard the holder announce once the database answered', + }); + expect(heardRemoved(lapsedApp, anchors), 'a peer was told of the removal of an app whose record had expired').to.equal(false); + }); }); }); From 0e5fae150f087d3172c01142525b4278722d8625 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:15:24 +0100 Subject: [PATCH 172/369] test(harness): 3308 installs every app on every node, the holder first Co-Authored-By: Claude Opus 5.5 --- .../3308-boot-holds-an-app-it-cannot-confirm.js | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index 516e628620..27c2825d48 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -18,9 +18,10 @@ * watched leaving it alone. The plain app is decided as soon as the database * answers: it is the canary that the announcement and the decisions run at all. * - * Every app asks for as many instances as the fleet has nodes and is on every node - * before the holder goes down, so none is placed anew while the holder is away. A - * standby may take over the mixed app's active-standby component in that time. + * Every app asks for as many instances as the fleet has nodes and is installed on + * every node, the holder first so it is the mixed app's primary, before the holder + * goes down: none is placed anew while the holder is away. A standby may take over + * the mixed app's active-standby component in that time. */ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; @@ -183,6 +184,11 @@ describe('boot holds an app whose location it cannot read', function () { await waitFor(async () => (await Promise.all([db, web, plain, lapsed].map(isUp))).every(Boolean), { timeout: 300000, interval: 3000, label: 'every app runs on the holder, the mixed app\'s db as its primary', }); + const peers = [...Array(NODES).keys()].filter((i) => i !== HOLDER); + for (const app of apps) { + // eslint-disable-next-line no-await-in-loop + await installOnNodes(env, app, peers); + } await waitFor(async () => (await Promise.all([mixedApp, plainApp, lapsedApp].map((appName) => installedInstanceIndices(env, appName)))) .every((holders) => holders.length === NODES), { timeout: 300000, interval: 5000, label: 'every app is on every node', From bc79d88b7b9d3e739c7062008494c9845c8ac3db Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:19:10 +0100 Subject: [PATCH 173/369] test(harness): 3308 records each step of its setup, so a node that does not hold the app fails at the assertion that names it Co-Authored-By: Claude Opus 5.5 --- ...308-boot-holds-an-app-it-cannot-confirm.js | 50 ++++++++++++------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index 27c2825d48..d2301015f1 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -99,7 +99,8 @@ describe('boot holds an app whose location it cannot read', function () { && client.getEventBuffer().some((e) => e.id > afterIds[i] && e.event === 'network:apprunning' && e.data?.ip?.startsWith(holder.ip) && (e.data?.apps ?? []).some((a) => a.name === appName))); // What the reconciler did with a component it was asked about after `afterId`: - // deferred it or started it, resolved once it has done one of the two. + // deferred it or started it, resolved once it has done one of the two; null when + // it did neither. const reconcilerAnswer = async (identifier, afterId) => { let answer = null; await waitFor(async () => { @@ -114,11 +115,16 @@ describe('boot holds an app whose location it cannot read', function () { return true; } return false; - }, { timeout: 120000, interval: 1000, label: `the reconciler answered for ${identifier}` }); + }, { timeout: 120000, interval: 1000, label: `the reconciler answered for ${identifier}` }).catch(() => {}); return answer; }; const cycles = async () => (await holder.getTestCounters())?.['masterSlave:cycles'] ?? 0; + // Each step below records what it saw rather than throwing, so a node that does + // not hold the app answers at the assertion that names what it did instead. + const heard = (name, predicate, timeout, afterId) => holder.waitForEvent(name, predicate, timeout, { afterId }) + .catch(() => null); + // FluxOS killed and held, the given containers stopped, the holder's records of // `keep` made to outlive the restart and those of `drop` deleted, and its reads of // the location collection refused from before FluxOS comes back. @@ -215,6 +221,8 @@ describe('boot holds an app whose location it cannot read', function () { let evaluatedWhileHeld; let skippedWhileHeld; let upWhileHeld; + let heldAtBoot; + let parkedAtDecision; before(async function () { this.timeout(900000); @@ -222,11 +230,11 @@ describe('boot holds an app whose location it cannot read', function () { ({ anchors, release } = await rebootUnreadable([db, web, plain], { keep: [mixedApp, plainApp], drop: [] })); // Held before boot's first read, which fails: the decision it holds comes at // least one retry interval after that. - await holder.holdCheckpoint(BEFORE_DECISION, mixedApp); + const holdable = await holder.holdCheckpoint(BEFORE_DECISION, mixedApp).then(() => true, () => false); const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); - await Promise.all([mixedApp, plainApp].map((appName) => holder.waitForEvent( - 'boot:appUnconfirmed', (d) => d.name === appName, 60000, { afterId: anchors[HOLDER] }, - ))); + heldAtBoot = (await Promise.all([mixedApp, plainApp].map((appName) => heard( + 'boot:appUnconfirmed', (d) => d.name === appName, 60000, anchors[HOLDER], + )))).every(Boolean); stoppedAtBoot = !(await Promise.all([db, web, plain].map(isUp))).some(Boolean); step('the location records unreadable: the reconciler is asked about every component'); @@ -243,11 +251,13 @@ describe('boot holds an app whose location it cannot read', function () { const evaluatedBefore = await holder.getDecisionCount('masterSlave:decision', db, 'evaluated'); failedReads = await release(); release = null; - await holder.waitForEvent('checkpoint:held', (d) => d.name === BEFORE_DECISION && d.key === mixedApp, 120000, { afterId: answeredFrom }); - await holder.waitForEvent('boot:appConfirmed', (d) => d.name === plainApp && d.valid === true, 120000, { afterId: answeredFrom }); - announcement = (await holder.waitForEvent( - 'app:running', (d) => (d.apps ?? []).some((a) => a.name === plainApp), 120000, { afterId: answeredFrom }, - ).catch(() => null))?.data ?? null; + parkedAtDecision = holdable && Boolean(await heard( + 'checkpoint:held', (d) => d.name === BEFORE_DECISION && d.key === mixedApp, 120000, answeredFrom, + )); + await heard('boot:appConfirmed', (d) => d.name === plainApp && d.valid === true, 120000, answeredFrom); + announcement = (await heard( + 'app:running', (d) => (d.apps ?? []).some((a) => a.name === plainApp), 120000, answeredFrom, + ))?.data ?? null; const cyclesFrom = await cycles(); await waitFor(async () => (await cycles()) >= cyclesFrom + 2, { timeout: 180000, interval: 2000, label: 'the election ran two passes while the mixed app was held', @@ -258,8 +268,8 @@ describe('boot holds an app whose location it cannot read', function () { upWhileHeld = (await Promise.all([db, web].map(isUp))).some(Boolean); step('the mixed app\'s decision released'); - await holder.releaseCheckpoint(BEFORE_DECISION, mixedApp); - await holder.waitForEvent('boot:appConfirmed', (d) => d.name === mixedApp && d.valid === true, 60000, { afterId: answeredFrom }); + if (holdable) await holder.releaseCheckpoint(BEFORE_DECISION, mixedApp); + await heard('boot:appConfirmed', (d) => d.name === mixedApp && d.valid === true, 60000, answeredFrom); }); after(async function () { @@ -278,6 +288,11 @@ describe('boot holds an app whose location it cannot read', function () { it('keeps an app it cannot confirm installed', function () { expect(installedWhileUnreadable, 'boot removed an app whose location it could not read').to.include.members([mixedApp, plainApp]); + expect(heldAtBoot, 'boot did not hold the apps whose location it could not read').to.equal(true); + }); + + it('holds the mixed app\'s decision at its checkpoint once the database answers', function () { + expect(parkedAtDecision, 'the mixed app\'s decision was not held, so the announcement and election below were not watched while it was undecided').to.equal(true); }); it('starts no component of an app it cannot confirm, when the reconciler is asked to', function () { @@ -329,13 +344,14 @@ describe('boot holds an app whose location it cannot read', function () { let installedWhileUnreadable; let removed; let installedAfter; + let heldAtBoot; before(async function () { this.timeout(900000); step('taking the node down again, the lapsed app\'s record deleted'); ({ anchors, release } = await rebootUnreadable([lapsed, plain], { keep: [plainApp], drop: [lapsedApp] })); const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); - await holder.waitForEvent('boot:appUnconfirmed', (d) => d.name === lapsedApp, 60000, { afterId: anchors[HOLDER] }); + heldAtBoot = Boolean(await heard('boot:appUnconfirmed', (d) => d.name === lapsedApp, 60000, anchors[HOLDER])); sweptAfterBoot = Boolean(await holder.waitForEvent('reconciler:swept', () => true, 120000, { afterId: settled.id }) .catch(() => null)); lapsedWhileUnreadable = await reconcilerAnswer(lapsed, settled.id); @@ -345,9 +361,8 @@ describe('boot holds an app whose location it cannot read', function () { const answeredFrom = holder.getLastEventId(); failedReads = await release(); release = null; - await holder.waitForEvent('boot:appConfirmed', (d) => d.name === lapsedApp && d.valid === false, 120000, { afterId: answeredFrom }); - removed = Boolean(await holder.waitForEvent('app:removed', (d) => d.name === lapsedApp, 120000, { afterId: answeredFrom }) - .catch(() => null)); + await heard('boot:appConfirmed', (d) => d.name === lapsedApp && d.valid === false, 120000, answeredFrom); + removed = Boolean(await heard('app:removed', (d) => d.name === lapsedApp, 120000, answeredFrom)); installedAfter = await installedNames(); }); @@ -363,6 +378,7 @@ describe('boot holds an app whose location it cannot read', function () { it('holds the app, installed and stopped, while its record cannot be read', function () { expect(sweptAfterBoot, 'no reconciler sweep ran while the records were unreadable, so nothing below was tested').to.equal(true); expect(installedWhileUnreadable, 'boot removed an app whose location it could not read').to.include(lapsedApp); + expect(heldAtBoot, 'boot did not hold the app whose location it could not read').to.equal(true); expect(lapsedWhileUnreadable, 'the reconciler started an app boot could not confirm').to.equal('deferred'); }); From d7511c6d5e17eab63e192a23c6aa942974a00053 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:23:55 +0100 Subject: [PATCH 174/369] test(harness): 3308's expired record is one the holder last renewed longer ago than its TTL, everywhere Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/db-client.js | 17 ++++------ ...308-boot-holds-an-app-it-cannot-confirm.js | 33 ++++++++++++------- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index ff3ebb21de..a8795cb404 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -64,6 +64,13 @@ export function dbClient(nodeNum) { await localDb.collection('ownaddress').deleteMany({}); }, + // The node keeps no record of when it was last alive, as on a node whose FluxOS + // never wrote one: its next boot reads as a first boot. + async forgetHeartbeat() { + const localDb = await db('local'); + await localDb.collection('nodestartuptracker').deleteOne({ _id: 'heartbeat' }); + }, + // The node's own geolocation record: what it decided about its address. async nodeGeolocation() { const localDb = await db('local'); @@ -113,16 +120,6 @@ export function dbClient(nodeNum) { // This node's view of where an app runs (the rows fluxapprunning messages // upsert and the empty-broadcast bug used to wipe). - // A location's expiry on this node: its rows for the app at that address are - // gone, as the TTL index removes them. Resolves to how many rows went. - async expireAppLocation(appName, ip) { - const globalDb = await db('appsGlobal'); - const host = String(ip).replace(/:\d+$/, '').replace(/\./g, '\\.'); - const { deletedCount } = await globalDb.collection('zelappslocation') - .deleteMany({ name: appName, ip: { $regex: `^${host}(:\\d+)?$` } }); - return deletedCount; - }, - // This node's rows for the app at that address stay unexpired for `ms` from now, // as a production record does across a restart the harness's compressed record // would not outlive. Resolves to how many rows were found. diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index d2301015f1..c1e5db90e9 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -10,7 +10,12 @@ * Here FluxOS is killed and held down, the apps' containers are stopped, and the * node's reads of the location collection are refused before FluxOS comes back. The * holder's records are made to outlive the restart, as a production record does and - * the harness's compressed one would not; a record meant to have expired is deleted. + * the harness's compressed one would not. + * + * A record that has expired is one the holder last renewed longer ago than its TTL, + * everywhere: the holder stays down that long. A node down that long whose boot + * history says so removes every app at once; one with no boot history - a FluxOS + * that never wrote one - decides app by app, which is the path tested here. * * The mixed app pairs an active-standby component, which only the election starts, * with a plain one boot starts. Its decision is held at a checkpoint once the @@ -39,6 +44,8 @@ import { dbClient } from '../framework/db-client.js'; import { isDaemonUp } from '../framework/syncthing-real.js'; import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; +import { loadSharedConfig, BOOT_DRIFT_MS } from '../framework/coupled-knobs.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; const HOLDER = 0; const NODES = 3; @@ -46,6 +53,8 @@ const APP_UID = 1000; const BEFORE_DECISION = 'boot:beforeLocationDecision'; // Longer than anything below, and far shorter than a production record's TTL. const RECORD_OUTLIVES_MS = 60 * 60 * 1000; +// Down this long, every location record of the holder has expired on every node. +const RECORDS_EXPIRE_MS = loadSharedConfig().fluxapps.locationTtlS * 1000 + BOOT_DRIFT_MS; function step(what) { console.log(`# 3308 ${new Date().toISOString()} ${what}`); @@ -126,9 +135,10 @@ describe('boot holds an app whose location it cannot read', function () { .catch(() => null); // FluxOS killed and held, the given containers stopped, the holder's records of - // `keep` made to outlive the restart and those of `drop` deleted, and its reads of - // the location collection refused from before FluxOS comes back. - async function rebootUnreadable(identifiers, { keep, drop }) { + // `keep` made to outlive the restart, and its reads of the location collection + // refused from before FluxOS comes back. `expire` holds it down until every record + // of it has expired, its boot history forgotten. + async function rebootUnreadable(identifiers, { keep = [], expire = false }) { await crashFluxos(holder.container, { hold: true }); for (const identifier of identifiers) { const [component, appName] = identifier.split('_'); @@ -141,10 +151,9 @@ describe('boot holds an app whose location it cannot read', function () { const kept = await holderDb.extendAppLocation(appName, holder.ip, RECORD_OUTLIVES_MS); if (kept < 1) throw new Error(`the holder has no location record of ${appName} to keep`); } - for (const appName of drop) { - // eslint-disable-next-line no-await-in-loop - const gone = await holderDb.expireAppLocation(appName, holder.ip); - if (gone < 1) throw new Error(`the holder has no location record of ${appName} to expire`); + if (expire) { + await sleepUnlessInfraDead(RECORDS_EXPIRE_MS); + await holderDb.forgetHeartbeat(); } const anchors = env.clients.map((client) => client?.getLastEventId()); const release = await failCommands(HOLDER + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); @@ -227,7 +236,7 @@ describe('boot holds an app whose location it cannot read', function () { before(async function () { this.timeout(900000); step('taking the node down, its containers stopped and its location reads refused'); - ({ anchors, release } = await rebootUnreadable([db, web, plain], { keep: [mixedApp, plainApp], drop: [] })); + ({ anchors, release } = await rebootUnreadable([db, web, plain], { keep: [mixedApp, plainApp] })); // Held before boot's first read, which fails: the decision it holds comes at // least one retry interval after that. const holdable = await holder.holdCheckpoint(BEFORE_DECISION, mixedApp).then(() => true, () => false); @@ -348,8 +357,8 @@ describe('boot holds an app whose location it cannot read', function () { before(async function () { this.timeout(900000); - step('taking the node down again, the lapsed app\'s record deleted'); - ({ anchors, release } = await rebootUnreadable([lapsed, plain], { keep: [plainApp], drop: [lapsedApp] })); + step('taking the node down again, for longer than its records live, its boot history forgotten'); + ({ anchors, release } = await rebootUnreadable([lapsed], { expire: true })); const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); heldAtBoot = Boolean(await heard('boot:appUnconfirmed', (d) => d.name === lapsedApp, 60000, anchors[HOLDER])); sweptAfterBoot = Boolean(await holder.waitForEvent('reconciler:swept', () => true, 120000, { afterId: settled.id }) @@ -389,7 +398,7 @@ describe('boot holds an app whose location it cannot read', function () { it('tells the network nothing of the removal, the network having dropped the record', async function () { this.timeout(120000); - await waitFor(() => heardRunning(plainApp, anchors), { + await waitFor(() => heardRunning(mixedApp, anchors), { timeout: 100000, interval: 2000, label: 'a peer heard the holder announce once the database answered', }); expect(heardRemoved(lapsedApp, anchors), 'a peer was told of the removal of an app whose record had expired').to.equal(false); From 6c51b19ea70cad82a103129bbb404f84d25d65c1 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:28:35 +0100 Subject: [PATCH 175/369] test(harness): 3308 reads the apps the holder holds from its own table, which no cache answers for Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/db-client.js | 8 ++++++++ .../tests/3308-boot-holds-an-app-it-cannot-confirm.js | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index a8795cb404..36b18b9b33 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -118,6 +118,14 @@ export function dbClient(nodeNum) { return localDb.collection('zelappsinformation').countDocuments({}); }, + // The names of the apps this node holds, from its own table: the route that + // lists them is cached, and this is not. + async localAppNames() { + const localDb = await db('appsLocal'); + const rows = await localDb.collection('zelappsinformation').find({}, { projection: { _id: 0, name: 1 } }).toArray(); + return rows.map((row) => row.name); + }, + // This node's view of where an app runs (the rows fluxapprunning messages // upsert and the empty-broadcast bug used to wipe). // This node's rows for the app at that address stay unexpired for `ms` from now, diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index c1e5db90e9..ed6608e453 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -100,7 +100,7 @@ describe('boot holds an app whose location it cannot read', function () { const isUp = async (identifier) => (await listAppContainers(holder.container)) .some((c) => c.name === `flux${identifier}` && c.status?.startsWith('Up')); - const installedNames = async () => ((await holder.getInstalledApps()).data ?? []).map((a) => a.name); + const installedNames = () => holderDb.localAppNames(); const heardRemoved = (appName, afterIds) => env.clients.some((client, i) => i !== HOLDER && client && client.getEventBuffer().some((e) => e.id > afterIds[i] && e.event === 'network:appremoved' && e.data?.name === appName && e.data?.ip?.startsWith(holder.ip))); From a9803cffd7062c25f2433b8fcd5dc478a8d96e80 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:31:22 +0100 Subject: [PATCH 176/369] test(harness): 3308's expired case hears the holder name the app that runs throughout Co-Authored-By: Claude Opus 5.5 --- .../runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index ed6608e453..e50a1a6847 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -398,7 +398,8 @@ describe('boot holds an app whose location it cannot read', function () { it('tells the network nothing of the removal, the network having dropped the record', async function () { this.timeout(120000); - await waitFor(() => heardRunning(mixedApp, anchors), { + // the plain app runs throughout, so boot does not check it and the holder names it + await waitFor(() => heardRunning(plainApp, anchors), { timeout: 100000, interval: 2000, label: 'a peer heard the holder announce once the database answered', }); expect(heardRemoved(lapsedApp, anchors), 'a peer was told of the removal of an app whose record had expired').to.equal(false); From 2004ccf4932abe4037e19d8ef8a2317a7e5fb7db Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:47:02 +0100 Subject: [PATCH 177/369] fix(appsync): a sync answer whose handler throws part way ends that peer's request A peer answers a state sync in several messages, the last marked done. A call that throws while one message is recorded - the broadcast verification, the location prune, a state event's store - loses the rest of that message, so the answer is not all recorded: the peer's request ends there and another peer is asked, as for a write that reports a partial result, and its later done is not counted. A failure in an answer whose request has already ended ends nothing. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxCommunication.js | 17 ++++ tests/unit/fluxCommunication.test.js | 105 ++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 720c115ce8..7452f6614a 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -150,6 +150,20 @@ async function handleTempSyncResponse(message, peerSocket) { } } +/** + * Ends the request of a peer whose sync response threw part way: what it sent is + * not all recorded, so another peer is asked, and the rest of its answer is not + * read. A peer whose response this node no longer wants has no request to end. + * @param {{key: string, connectionId?: number}} peerSocket + * @param {string} handler Names the handler in the log. + * @returns {void} + */ +function endUnrecordedRequest(peerSocket, handler) { + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + log.warn(`${handler} - the response was not all recorded, ending ${peerSocket.key}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerSocket.key); +} + async function handleAppRunningSyncResponse(message, peerSocket) { try { if (!message.data || message.data.type !== 'fluxapprunningsync') return; @@ -280,6 +294,7 @@ async function handleAppRunningSyncResponse(message, peerSocket) { } } catch (error) { log.error(error); + endUnrecordedRequest(peerSocket, 'handleAppRunningSyncResponse'); } } @@ -319,6 +334,7 @@ async function handleAppInstallingSyncResponse(message, peerSocket) { } } catch (error) { log.error(error); + endUnrecordedRequest(peerSocket, 'handleAppInstallingSyncResponse'); } } @@ -358,6 +374,7 @@ async function handleAppInstallingErrorsSyncResponse(message, peerSocket) { } } catch (error) { log.error(error); + endUnrecordedRequest(peerSocket, 'handleAppInstallingErrorsSyncResponse'); } } diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index 022fc93720..7afed70a1c 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -2575,6 +2575,111 @@ describe('fluxCommunication tests', () => { }); }); + // A call that throws part way through a message loses the rest of it, so the + // answer is not all recorded: the peer's request ends and another is asked, + // and the done that follows is not counted. + describe('a response whose calls threw', () => { + let writeFailures; + let verdict; + const onWriteFailed = (peerKey) => writeFailures.push(peerKey); + const envelope = () => ({ version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig' }); + const broadcast = () => ({ ...envelope(), data: { ip: '1.2.3.4:16127', name: 'a', broadcastedAt: Date.now() } }); + const running = () => ({ + type: 'apprunning', envelope: envelope(), data: { version: 2, ip: '1.2.3.4:16127', broadcastedAt: Date.now(), apps: [{ name: 'a' }] }, + }); + const sigterm = () => ({ type: 'sigterm', envelope: envelope(), data: { ip: '1.2.3.4:16127', broadcastedAt: Date.now() } }); + const respond = (handler, type, messages, done) => fluxCommunication[handler]({ data: { type, messages, done } }, PEER_SOCKET); + + beforeEach(() => { + writeFailures = []; + verdict = fluxCommunicationUtils.VerifyResult.OK; + appSyncEvents.on(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + sinon.stub(fluxCommunicationUtils, 'verifyBroadcasts').callsFake(async (items) => items.map(() => ({ result: verdict }))); + sinon.stub(messageStore, 'storeBatchAppRunningMessages').resolves({ stored: 1, writeFailed: false }); + sinon.stub(messageStore, 'storeBatchAppInstallingMessages').resolves({ stored: 1, writeFailed: false }); + sinon.stub(messageStore, 'storeBatchAppInstallingErrorMessages').resolves({ stored: 1, writeFailed: false }); + sinon.stub(messageStore, 'storeAppStateEvent').resolves(); + sinon.stub(messageStore, 'expireLocationsForSigterm').resolves(); + }); + + afterEach(() => { + appSyncEvents.removeListener(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + }); + + [ + ['appinstalling', 'handleAppInstallingSyncResponse', 'fluxappinstallingsync', broadcast], + ['apperrors', 'handleAppInstallingErrorsSyncResponse', 'fluxappinstallingerrorssync', broadcast], + ['apprunning', 'handleAppRunningSyncResponse', 'fluxapprunningsync', running], + ].forEach(([syncType, handler, type, event]) => { + it(`ends the ${syncType} request when a broadcast it sent cannot be verified for want of the database`, async () => { + verdict = fluxCommunicationUtils.VerifyResult.UNREADABLE; + + await respond(handler, type, [event()], true); + + expect(writeFailures, 'a response the node could not verify left the request open').to.deep.equal([PEER]); + expect(completions, 'a response the node could not verify was counted').to.deep.equal([]); + }); + + it(`a canary: the same ${syncType} response verified and stored is counted`, async () => { + await respond(handler, type, [event()], true); + + expect(writeFailures).to.deep.equal([]); + expect(completions).to.deep.equal([{ syncType, peerKey: PEER }]); + }); + }); + + it('ends the appinstalling request when its store throws', async () => { + messageStore.storeBatchAppInstallingMessages.rejects(new Error('operation exceeded time limit')); + + await respond('handleAppInstallingSyncResponse', 'fluxappinstallingsync', [broadcast()], true); + + expect(writeFailures).to.deep.equal([PEER]); + expect(completions).to.deep.equal([]); + }); + + it('ends the apprunning request when pruning its locations throws', async () => { + messageStore.pruneAppRunningLocations.rejects(new Error('operation exceeded time limit')); + + await respond('handleAppRunningSyncResponse', 'fluxapprunningsync', [running()], true); + + expect(messageStore.pruneAppRunningLocations.called, 'the fixture never reached the prune').to.be.true; + expect(writeFailures).to.deep.equal([PEER]); + expect(completions).to.deep.equal([]); + }); + + it('ends the apprunning request when storing a state event throws', async () => { + messageStore.storeAppStateEvent.rejects(new Error('operation exceeded time limit')); + + await respond('handleAppRunningSyncResponse', 'fluxapprunningsync', [running(), sigterm()], true); + + expect(messageStore.storeAppStateEvent.called, 'the fixture never reached the state event').to.be.true; + expect(writeFailures).to.deep.equal([PEER]); + expect(completions).to.deep.equal([]); + }); + + it('a canary: the same apprunning response with its state event stored is counted', async () => { + await respond('handleAppRunningSyncResponse', 'fluxapprunningsync', [running(), sigterm()], true); + + expect(messageStore.storeAppStateEvent.called, 'the fixture never reached the state event').to.be.true; + expect(writeFailures).to.deep.equal([]); + expect(completions).to.deep.equal([{ syncType: 'apprunning', peerKey: PEER }]); + }); + + // The request this answer belonged to ended while it was being stored - the + // peer reconnected and was asked again - so a failure here ends nothing. + it('ends nothing when the request it answered has already ended', async () => { + messageStore.storeBatchAppInstallingMessages.callsFake(async () => { + wanted = () => false; + throw new Error('operation exceeded time limit'); + }); + + await respond('handleAppInstallingSyncResponse', 'fluxappinstallingsync', [broadcast()], true); + + expect(messageStore.storeBatchAppInstallingMessages.called, 'the fixture never reached the store').to.be.true; + expect(writeFailures, 'a failure ended a request it did not belong to').to.deep.equal([]); + }); + }); + describe('a chunk takes its place in the queue when it arrives', () => { const chunkFor = (marker) => ({ data: { type: 'fluxapprunningsync', messages: [{ marker }], done: false }, From 51be2a083aff91a925a3fbe17d801be4fc7fda5d Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:47:03 +0100 Subject: [PATCH 178/369] fix(apps): the test event of a removal is published once the removal has finished, the app's row included Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appUninstaller.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 292461646e..bfbb12d3ab 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -1051,8 +1051,6 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (onComponentRemoved) onComponentRemoved(identifier); } - fluxEventBus.publish('app:removed', { name: appName }); - if (sendMessage) { await announceRemoval(appName); announced = true; @@ -1152,6 +1150,9 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } } + // Published once the removal has finished, the app's row included. + fluxEventBus.publish('app:removed', { name: appName }); + const appRemovalResponseDone = messageHelper.createSuccessMessage(`Removal step done. Result: Flux App ${appName} was successfuly removed`); log.info(appRemovalResponseDone); From 9d2cd4793617cdc537b450e2459bee3004de7b76 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:48:02 +0100 Subject: [PATCH 179/369] test(harness): 3304, a sync answer whose location prune throws is not counted Co-Authored-By: Claude Opus 5.5 --- ...r-the-node-cannot-record-is-not-counted.js | 31 +++++++++++++------ 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js index d8e7fa2983..484f2a5f73 100644 --- a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js +++ b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js @@ -5,14 +5,16 @@ import { waitForDaemonReady, waitForNodeStatus, waitFor, } from '../framework/wait.js'; import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; -import { failWrites } from '../framework/mongo-control.js'; +import { failCommands, failWrites } from '../framework/mongo-control.js'; import { dbClient } from '../framework/db-client.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // A node joining the network asks a peer for the network's app state, and the peer // answers in several messages, only the last saying it is done. Here the joining -// node's database fails the write of the first message, and the node must not -// count that peer as having answered: it ends the request and asks another peer. +// node's database fails a call made while it records the first message - a write +// that reports it stored part of the batch, or the prune of old location rows, +// which throws - and the node must not count that peer as having answered: it ends +// the request and asks another peer. // // The harness keeps one sync request open at a time (appSyncMinCompletions), so a // peer counted despite the failed write leaves one peer asked, and a peer replaced @@ -33,7 +35,18 @@ function step(what) { console.log(`# 3304 ${new Date().toISOString()} ${what}`); } -describe('a sync answer the node cannot record is not counted', function () { +const FAILURES = [ + { + what: 'a location write the database fails', + arm: (nodeNum) => failWrites(nodeNum, 'globalzelapps', 'zelappslocation', { times: 1 }), + }, + { + what: 'a location prune the database fails, which throws', + arm: (nodeNum) => failCommands(nodeNum, 'globalzelapps', 'zelappslocation', { commands: ['delete'], times: 1 }), + }, +]; + +FAILURES.forEach(({ what, arm }) => describe(`a sync answer the node cannot record is not counted: ${what}`, function () { let env; dumpLogsOnFailure(() => env); const appName = `e2esyncwrite${Date.now()}`; @@ -66,8 +79,8 @@ describe('a sync answer the node cannot record is not counted', function () { }); eventsHeld = (await dbClient(1).appStateEvents({ type: 'apprunning' })).length; - step('arming the joining node\'s first location write to fail'); - release = await failWrites(JOINING + 1, 'globalzelapps', 'zelappslocation', { times: 1 }); + step(`arming ${what}`); + release = await arm(JOINING + 1); joining = await env.startNode(JOINING); await waitForDaemonReady(joining); await waitForNodeStatus(joining, (d) => d.confirmed === true, 60000); @@ -85,10 +98,10 @@ describe('a sync answer the node cannot record is not counted', function () { expect(eventsHeld, 'the answer fits one message, so no early message could fail').to.be.above(ANSWER_BATCH); }); - it('had one of its location writes refused by the database', async function () { + it('had the call refused by the database', async function () { const failed = await release(); release = null; - expect(failed, 'the database failed none of the joining node\'s writes, so nothing below was tested').to.equal(1); + expect(failed, 'the database failed none of the joining node\'s calls, so nothing below was tested').to.equal(1); }); it('asks another peer in place of the one whose answer it could not record', function () { @@ -109,4 +122,4 @@ describe('a sync answer the node cannot record is not counted', function () { && e.data.syncType === 'apprunning' && e.data.peer === failedAt.data.peer); expect(counted, 'the peer whose answer was not recorded was counted').to.deep.equal([]); }); -}); +})); From 33b90fa2ab3270a5611022bac05e85ce0a07f43c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:56:18 +0100 Subject: [PATCH 180/369] fix(appsync): a hash whose message the node already stores is neither backed off nor given up on The retry ladder spaces out asking peers for a message no peer has sent. A hash whose message is stored waits on no peer: its registry row is written from the stored copy on the next pass, so the ladder does not apply to it. Co-Authored-By: Claude Opus 5.5 --- .../appMessaging/appHashSyncService.js | 12 ++++ tests/unit/appHashSyncService.test.js | 57 +++++++++++++++++++ 2 files changed, 69 insertions(+) diff --git a/ZelBack/src/services/appMessaging/appHashSyncService.js b/ZelBack/src/services/appMessaging/appHashSyncService.js index d38556cdb5..e46f2e8e28 100644 --- a/ZelBack/src/services/appMessaging/appHashSyncService.js +++ b/ZelBack/src/services/appMessaging/appHashSyncService.js @@ -790,10 +790,22 @@ async function syncMissingHashes(options = {}) { // Update backoff for unresolved hashes const finalMissing = await getMissingHashes({ force: false, currentHeight }); + // A hash whose message this node already stores waits on no peer: the next pass + // writes its registry row from the stored copy, so it is neither backed off nor + // given up on. + const stored = new Set(); + for (let i = 0; i < finalMissing.length; i += CHUNK_SIZE) { + const hashValues = finalMissing.slice(i, i + CHUNK_SIZE).map((h) => h.hash); + // eslint-disable-next-line no-await-in-loop + const found = await dbHelper.findInDatabase(appsGlobalDb, globalAppsMessages, { hash: { $in: hashValues } }, { projection: { _id: 0, hash: 1 } }); + found.forEach((m) => stored.add(m.hash)); + } + let unreachable = 0; let minNextRetry = null; const backoffOps = []; for (const hashDoc of finalMissing) { + if (stored.has(hashDoc.hash)) continue; if (currentHeight - hashDoc.retryFromHeight > HASH_EXPIRY_BLOCKS) { // eslint-disable-next-line no-await-in-loop await messageVerifier.appHashHasMessageNotFound(hashDoc.hash); diff --git a/tests/unit/appHashSyncService.test.js b/tests/unit/appHashSyncService.test.js index 3ebfa5e322..ee82ef3c14 100644 --- a/tests/unit/appHashSyncService.test.js +++ b/tests/unit/appHashSyncService.test.js @@ -6,6 +6,8 @@ const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileg const { Readable } = require('stream'); const config = require('config'); +const MESSAGES = config.database.appsglobal.collections.appsMessages; + function makeStreamResponse(data) { const json = JSON.stringify({ status: 'success', data }); const stream = new Readable({ read() { this.push(json); this.push(null); } }); @@ -834,6 +836,7 @@ describe('appHashSyncService tests', () => { const { recentRetry, oldRetry } = makeExpiryTestHashes(); const both = [recentRetry, oldRetry]; dbHelperStub.findInDatabase.resolves(both); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([]); advanceDateOnDelay(); try { @@ -854,6 +857,7 @@ describe('appHashSyncService tests', () => { dbHelperStub.findInDatabase.onFirstCall().resolves([recentRetry, oldRetry, ...padding]); dbHelperStub.findInDatabase.resolves([recentRetry, oldRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([]); serviceHelperStub.axiosGet.callsFake((url) => { if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse([])); @@ -869,6 +873,59 @@ describe('appHashSyncService tests', () => { assertExpiryBehavior(); }); + + // A hash whose message this node holds waits on no peer: the next pass writes its + // registry row from the stored copy, so the ladder that spaces out asking peers + // does not apply to it. + describe('a hash whose message is already stored', () => { + const backedOff = (hash) => collectionStub.bulkWrite.getCalls().some((call) => call.args[0] + .some((op) => op.updateOne?.filter?.hash === hash && op.updateOne.update.$set.syncAttempts !== undefined)); + + it('is not backed off', async () => { + const { recentRetry } = makeExpiryTestHashes(); + dbHelperStub.findInDatabase.resolves([recentRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([{ hash: recentRetry.hash }]); + + advanceDateOnDelay(); + try { + await appHashSyncService.syncMissingHashes({ currentHeight: 2555000 }); + } finally { + Date.now.restore(); + } + + expect(backedOff(recentRetry.hash), 'a hash whose message is stored was backed off').to.equal(false); + }); + + it('a canary: the same hash with no message stored is backed off', async () => { + const { recentRetry } = makeExpiryTestHashes(); + dbHelperStub.findInDatabase.resolves([recentRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([]); + + advanceDateOnDelay(); + try { + await appHashSyncService.syncMissingHashes({ currentHeight: 2555000 }); + } finally { + Date.now.restore(); + } + + expect(backedOff(recentRetry.hash)).to.equal(true); + }); + + it('is not given up on, however long it has waited', async () => { + const { oldRetry } = makeExpiryTestHashes(); + dbHelperStub.findInDatabase.resolves([oldRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([{ hash: oldRetry.hash }]); + + advanceDateOnDelay(); + try { + await appHashSyncService.syncMissingHashes({ currentHeight: 2555000 }); + } finally { + Date.now.restore(); + } + + expect(messageVerifierStub.appHashHasMessageNotFound.called, 'a hash whose message is stored was given up on').to.equal(false); + }); + }); }); describe('processMessages via bulk fetch', () => { From 5f91b4f6e9407a7e0fcbc46b598657ec77251962 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:57:46 +0100 Subject: [PATCH 181/369] test(harness): 3311, a late message whose registry row could not be written gets it on the next pass Co-Authored-By: Claude Opus 5.5 --- ...-could-not-be-written-gets-it-next-pass.js | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js diff --git a/test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js b/test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js new file mode 100644 index 0000000000..1d8efad6c7 --- /dev/null +++ b/test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js @@ -0,0 +1,125 @@ +/* + * A running node fetches in bulk the app messages missing behind its payment records, + * stores them, and then writes the registry row of each app they belong to before it + * marks their hashes found. Here the database refuses the first of those row writes: + * the messages are stored, the rows are not, and the hashes stay unmarked. + * + * A hash whose message the node already stores waits on no peer, so the retry ladder + * that spaces out asking peers does not apply to it: the next pass writes the rows from + * the stored copies. The node must hold every app at its newest message within a few + * passes, and must never have backed off one of those hashes. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { waitForDaemonReady, waitForOrchestratorState, waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const NODES = 4; +// the running node: held back until the fleet is up +const SUBJECT = NODES - 1; +const LATE_APPS = 10; +const UPDATES = 54; + +const unresolved = (entry) => ({ ...entry, message: false }); + +function step(what) { + console.log(`# 3311 ${new Date().toISOString()} ${what}`); +} + +describe('a late message whose registry row could not be written gets it on the next pass', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + // the newest message of each app the late log brings, by name + const governing = new Map(); + let lateHashes; + let release; + let failedWrites; + let bulkFetched; + const db = () => dbClient(SUBJECT + 1); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + deferredNodes: 1, + // a pass comes every few blocks + configOverrides: { fluxapps: { hashSyncFallbackRecheckBlocks: 2 } }, + }); + await bootAndPeer(env); + const base = env.initialHeight - 2000; + + const subject = await env.startNode(SUBJECT); + await waitForDaemonReady(subject); + await env.startDiscovery([SUBJECT]); + await waitForOrchestratorState(subject, 'READY', 600000); + + // the late log: enough messages to take the bulk path + const late = []; + for (let k = 0; k < LATE_APPS; k += 1) { + const name = `e2eRowLate${k}x${stamp}`; + // eslint-disable-next-line no-await-in-loop + let current = await buildSeedableApp({ env, name, height: base + 100 + (k * 60) }); + late.push(current); + for (let j = 1; j <= UPDATES; j += 1) { + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableUpdate(current, (spec) => { spec.description = `${name} revision ${j}`; }, { height: base + 100 + (k * 60) + j }); + late.push(current); + } + governing.set(name, current.hash); + } + expect(late.length, 'enough messages late to take the bulk path').to.be.above(500); + lateHashes = late.map((m) => m.hashEntry.hash); + + const anchor = subject.getLastEventId(); + step(`seeding ${late.length} late messages, the subject's first registry row write refused`); + release = await failCommands(SUBJECT + 1, 'globalzelapps', 'zelappsinformation', { commands: ['update'], times: 1 }); + await Promise.all(env.clients.map(async (client, i) => { + if (!client || i === SUBJECT) return; + await dbClient(i + 1).seedPermanentMessages(late.map((m) => m.permanentMessage)); + await dbClient(i + 1).seedAppHashes(late.map((m) => m.hashEntry)); + })); + await db().seedAppHashes(late.map((m) => unresolved(m.hashEntry))); + bulkFetched = await subject.waitForEvent('hashSync:bulkFetched', (d) => d.processed > 0, 600000, { afterId: anchor }) + .catch(() => null); + step('fetched in bulk'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('took the late messages in bulk', function () { + expect(bulkFetched, 'the node never fetched the late messages in bulk, so nothing below was tested').to.not.equal(null); + }); + + it('had a registry row write refused by the database', async function () { + failedWrites = await release(); + release = null; + expect(failedWrites, 'the database refused none of the node\'s registry writes, so nothing below was tested').to.equal(1); + }); + + it('holds every late app at its newest message within a few passes', async function () { + this.timeout(180000); + const names = [...governing.keys()]; + await waitFor(async () => { + const held = await Promise.all(names.map(async (name) => (await db().globalAppSpec(name))?.hash)); + return held.every((hash, i) => hash === governing.get(names[i])); + }, { timeout: 150000, interval: 3000, label: 'every late app held at its newest message' }); + }); + + it('never backed off a hash whose message it already stored', async function () { + const records = await db().appHashRecords({ hash: { $in: lateHashes } }); + const backedOff = records.filter((r) => (r.syncAttempts ?? 0) > 0).map((r) => r.hash); + expect(records.length, 'the node holds no record of the late hashes').to.equal(lateHashes.length); + expect(backedOff, 'a hash whose message the node stored was backed off as if no peer had it').to.deep.equal([]); + }); +}); From 8b8fd0638e80bb94c31ba410009d72fd9b359aba Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:07:19 +0100 Subject: [PATCH 182/369] fix(apps): a hard redeploy refused by another removal answers busy, the app as it was Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appRebuild.js | 16 ++++++++++------ tests/unit/appRebuild.test.js | 11 +++++++++++ 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appRebuild.js b/ZelBack/src/services/appLifecycle/appRebuild.js index 3b103b2518..bdaae867e4 100644 --- a/ZelBack/src/services/appLifecycle/appRebuild.js +++ b/ZelBack/src/services/appLifecycle/appRebuild.js @@ -13,19 +13,23 @@ const { InstallOutcome } = require('../utils/installOutcome'); /** * Remove the app and install it again at appSpecs. * - * A removal that does not finish leaves the app as it was, with its row, for the - * reconciler. Once the removal has finished this node holds nothing of the app, so - * an install that does not complete is told to the network, once, here. + * Nothing is installed unless the removal finished. A node busy with another removal + * leaves the app as it was; a removal that stopped part way is finished by the + * uninstaller, which removes the whole app and tells the network. Once the removal + * has finished this node holds nothing of the app, so an install that does not + * complete is told to the network, once, here. * @param {object} appSpecs The specification to install. * @param {object} [res] A response stream the caller holds. - * @returns {Promise} The InstallOutcome; FAILED when the removal did not finish. + * @returns {Promise} The InstallOutcome; BUSY when another removal held the + * node, FAILED when the removal did not finish. */ async function rebuildApp(appSpecs, res) { const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); if (removal !== RemovalOutcome.REMOVED && removal !== RemovalOutcome.NOT_INSTALLED) { log.warn(`appRebuild - ${appSpecs.name} was not removed (${removal}), so it is not reinstalled`); - fluxEventBus.publish('app:rebuilt', { name: appSpecs.name, outcome: InstallOutcome.FAILED }); - return InstallOutcome.FAILED; + const outcome = removal === RemovalOutcome.BUSY ? InstallOutcome.BUSY : InstallOutcome.FAILED; + fluxEventBus.publish('app:rebuilt', { name: appSpecs.name, outcome }); + return outcome; } let outcome; diff --git a/tests/unit/appRebuild.test.js b/tests/unit/appRebuild.test.js index 4bcaa62d9f..cb46a8b177 100644 --- a/tests/unit/appRebuild.test.js +++ b/tests/unit/appRebuild.test.js @@ -63,6 +63,17 @@ describe('appRebuild tests', () => { sinon.assert.calledOnceWithExactly(uninstaller.announceRemoval, 'myapp'); }); + // Another removal held the node, so this one touched nothing: the caller can ask + // again, and the app is as it was. + it('answers busy, installing nothing and telling the network nothing, when another removal holds the node', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.BUSY); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.BUSY); + + sinon.assert.notCalled(installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + it('installs nothing and tells the network nothing when the removal did not finish', async () => { uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); From eeb690fd9b831896b2cb3961c18b7578358f472b Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:11:02 +0100 Subject: [PATCH 183/369] fix(apps): a removal run again to finish it waits while another install or removal holds the node A forced removal takes the node's removal lock without asking and clears it when it ends, so run beside another removal it would release that removal's lock part way through. The run that finishes a part-removed app now waits out any operation holding the node and runs on a later pass. Co-Authored-By: Claude Opus 5.5 --- .../services/appLifecycle/appUninstaller.js | 5 +++ tests/unit/appUninstaller.test.js | 39 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index bfbb12d3ab..c502515e01 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -77,6 +77,11 @@ function finishRemoval(appName, force, announce) { log.warn(`${appName} was not fully removed; held stopped and unannounced until it is`); // first run once the removal that stopped has released the node's removal lock serviceHelper.delay(BOOT_STEP_RETRY_MS).then(() => untilSucceeds(`Removal of ${appName}`, async () => { + // Another install or removal holding the node is waited out, never run beside: + // a forced removal takes the removal lock without asking and clears it when it + // ends. Asked in the same turn as the removal starts, so nothing starts between. + const holder = globalState.operationHolding(); + if (holder) throw new Error(`another ${holder} holds the node`); // eslint-disable-next-line no-use-before-define const outcome = await removeAppLocally(appName, null, force, true, announce); if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 6ea0de208a..1838bb9348 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -616,9 +616,28 @@ describe('appUninstaller tests', () => { receiveOnlySyncthingAppsCache: new Map(), folderHoldings: new Map(), seedMarks: new Map(), + // the two operations this suite sets, named as the module names them + operationHolding() { + if (this.removalInProgress) return 'removal'; + if (this.installationInProgress) return 'installation'; + return null; + }, }; return proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { '../utils/globalState': globalStateStub, + // runs the step again after a moment until it succeeds, as the real one does after its wait + '../utils/untilSucceeds': { + BOOT_STEP_RETRY_MS: 15000, + untilSucceeds: async (what, step) => { + for (;;) { + try { + return await step(); // eslint-disable-line no-await-in-loop + } catch { + await new Promise((resolve) => { setTimeout(resolve, 10); }); // eslint-disable-line no-await-in-loop + } + } + }, + }, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -795,6 +814,26 @@ describe('appUninstaller tests', () => { sinon.assert.calledOnce(deleteRow); }); + // A forced removal takes the node's removal lock without asking and clears it + // when it ends, so one run beside another removal would release that removal's + // lock part way through. The rerun waits for the node instead. + it('runs the removal again only once no other install or removal holds the node', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + + globalStateStub.installationInProgress = true; + await uninstaller.removeAppLocally('testapp', res, true); + // time for the rerun to have run, had it not waited + await new Promise((resolve) => { setTimeout(resolve, 100); }); + sinon.assert.calledOnce(deleteRow); + expect(globalStateStub.installationInProgress, 'the rerun cleared another operation\'s hold').to.equal(true); + + globalStateStub.installationInProgress = false; + await settle(() => deleteRow.calledTwice && !globalStateStub.departingApps.has('testapp')); + sinon.assert.calledTwice(deleteRow); + }); + it('runs one removal again for an app however often it is handed over', async () => { let finishRow; const deleteRow = sinon.stub().resolves(); From 01e5080f27b9e1e9dbf2a307db82f0d6a5e380bb Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:15:31 +0100 Subject: [PATCH 184/369] fix(db): every index build repairs the duplicate rows that block it, the build after a conflicting index is dropped included Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 52 ++++++++++++++++++------------ tests/unit/dbHelperIndexes.test.js | 20 ++++++++++++ 2 files changed, 52 insertions(+), 20 deletions(-) diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index 80de05c10b..7cc48b18af 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -248,7 +248,8 @@ async function dedupeByKey(collection, spec, options = {}) { * IndexKeySpecsConflict) is dropped by its real name and recreated; * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the - * index rather than running degraded without it; + * index rather than running degraded without it. Every build goes this way, + * the one after a conflicting index is dropped included (buildIndex); * - anything else rethrows. * * The rethrow is deliberate. Index setup runs in serviceManager's @@ -266,30 +267,41 @@ async function dedupeByKey(collection, spec, options = {}) { * @param {(collection: object, spec: object, options: object) => Promise} [recover] * run when a unique build is blocked by existing duplicate rows */ -async function ensureIndex(collection, spec, options = {}, recover = null) { +/** + * Builds one index; a unique build blocked by rows that already violate it runs + * `recover` and builds again. + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key + * @param {object} options - the index options + * @param {Function|null} recover + * @returns {Promise} + */ +async function buildIndex(collection, spec, options, recover) { try { await collection.createIndex(spec, options); } catch (err) { - const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); - if (conflict) { - const specKeys = JSON.stringify(spec); - const indexes = await collection.listIndexes().toArray(); - const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); - if (match?.name) { - log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); - await collection.dropIndex(match.name); - } - await collection.createIndex(spec, options); - return; - } const duplicate = err && (err.code === 11000 || err.codeName === 'DuplicateKey'); - if (duplicate && recover) { - const removed = await recover(collection, spec, options); - log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); - await collection.createIndex(spec, options); - return; + if (!duplicate || !recover) throw err; + const removed = await recover(collection, spec, options); + log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); + await collection.createIndex(spec, options); + } +} + +async function ensureIndex(collection, spec, options = {}, recover = null) { + try { + await buildIndex(collection, spec, options, recover); + } catch (err) { + const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); + if (!conflict) throw err; + const specKeys = JSON.stringify(spec); + const indexes = await collection.listIndexes().toArray(); + const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); + if (match?.name) { + log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); + await collection.dropIndex(match.name); } - throw err; + await buildIndex(collection, spec, options, recover); } } diff --git a/tests/unit/dbHelperIndexes.test.js b/tests/unit/dbHelperIndexes.test.js index 10be911079..4fa392b22e 100644 --- a/tests/unit/dbHelperIndexes.test.js +++ b/tests/unit/dbHelperIndexes.test.js @@ -74,6 +74,26 @@ describe('dbHelper ensureIndex', () => { sinon.assert.notCalled(logErrorSpy); }); + // A node holding the old non-unique index of the same key also holds the rows it + // never refused: the build after the drop meets them, and repairs them the same way. + it('repairs duplicate rows that block the build after a conflicting index is dropped', async () => { + const conflict = new Error('Index with name: hash_1 already exists with different options'); + conflict.codeName = 'IndexOptionsConflict'; + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(conflict); + createIndex.onSecondCall().rejects(duplicateError()); + createIndex.onThirdCall().resolves('hash_1'); + const collection = stubCollection(createIndex); + collection.listIndexes.returns({ toArray: async () => [{ key: { hash: 1 }, name: 'hash_1' }] }); + const recover = sinon.stub().resolves(2); + + await ensureIndex(collection, { hash: 1 }, { unique: true }, recover); + + sinon.assert.calledOnceWithExactly(collection.dropIndex, 'hash_1'); + sinon.assert.calledOnceWithExactly(recover, collection, { hash: 1 }, { unique: true }); + sinon.assert.calledThrice(createIndex); + }); + it('runs the recovery strategy and rebuilds when duplicate rows block a unique index', async () => { // The node ends up WITH the index, not running degraded without it. const createIndex = sinon.stub(); From 687d3b602342b807b5131ca96ef68295b5d24f16 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:18:20 +0100 Subject: [PATCH 185/369] fix(dos): a residential node's recorded departure time never moves backwards Co-Authored-By: Claude Opus 5.5 --- .../src/services/residentialNodeDosService.js | 3 +- tests/unit/residentialNodeDosService.test.js | 57 ++++++++++++++++++- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/residentialNodeDosService.js b/ZelBack/src/services/residentialNodeDosService.js index 36504f8577..b119e90233 100644 --- a/ZelBack/src/services/residentialNodeDosService.js +++ b/ZelBack/src/services/residentialNodeDosService.js @@ -580,7 +580,8 @@ async function persistLastEvacuationAt(now) { database, startupCollection, { _id: SETTLE_MARKER_KEY }, - { $set: { lastEvacuationAt: now } }, + // never backwards: a retried older time does not replace a newer one + { $max: { lastEvacuationAt: now } }, { upsert: true }, ); if (unpersistedEvacuationAt === now) unpersistedEvacuationAt = null; diff --git a/tests/unit/residentialNodeDosService.test.js b/tests/unit/residentialNodeDosService.test.js index 2c17575c66..8589b1c336 100644 --- a/tests/unit/residentialNodeDosService.test.js +++ b/tests/unit/residentialNodeDosService.test.js @@ -20,6 +20,7 @@ describe('residentialNodeDosService tests', () => { let benchmarkServiceStub; let dbHelperStub; let markerStore; + let applyUpdate; let globalStateStub; let deps; let installedApps; @@ -101,12 +102,17 @@ describe('residentialNodeDosService tests', () => { // The settle marker lives in mongo, so the double has to remember it across // a reload - surviving a restart is the whole point of that field. markerStore = new Map(); + applyUpdate = (query, update) => { + const held = { _id: query._id, ...(markerStore.get(query._id) || {}), ...update.$set }; + Object.entries(update.$max || {}).forEach(([field, value]) => { + if (held[field] === undefined || value > held[field]) held[field] = value; + }); + markerStore.set(query._id, held); + }; dbHelperStub = { databaseConnection: sinon.stub().returns({ db: sinon.stub().returns({}) }), findOneInDatabase: sinon.stub().callsFake(async (db, coll, query) => markerStore.get(query._id) || null), - findOneAndUpdateInDatabase: sinon.stub().callsFake(async (db, coll, query, update) => { - markerStore.set(query._id, { _id: query._id, ...(markerStore.get(query._id) || {}), ...update.$set }); - }), + findOneAndUpdateInDatabase: sinon.stub().callsFake(async (db, coll, query, update) => applyUpdate(query, update)), findOneAndDeleteInDatabase: sinon.stub().callsFake(async (db, coll, query) => { markerStore.delete(query._id); }), @@ -621,6 +627,51 @@ describe('residentialNodeDosService tests', () => { expect(markerStore.get('residentialDos').lastEvacuationAt).to.be.a('number'); }); + it('keeps the newer departure time when an older one it could not record is written again', async () => { + installedApps = [{ name: 'someapp' }]; + markerStore.set('residentialDos', windowServed()); + const wall = sinon.stub(Date, 'now'); + try { + dbHelperStub.findOneAndUpdateInDatabase.onFirstCall().rejects(new Error('marker write failed')); + wall.returns(1000000); + service.noteEvacuated('someapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + wall.returns(2000000); + service.noteEvacuated('otherapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(markerStore.get('residentialDos').lastEvacuationAt, 'the newer departure was not recorded').to.equal(2000000); + + await service.enforceResidentialPolicy(deps); + } finally { + wall.restore(); + } + + expect(markerStore.get('residentialDos').lastEvacuationAt, 'an older departure replaced a newer one').to.equal(2000000); + }); + + it('keeps the newer departure time when an older write lands after it', async () => { + markerStore.set('residentialDos', windowServed()); + let landOlder; + dbHelperStub.findOneAndUpdateInDatabase.onFirstCall().callsFake(async (db, coll, query, update) => { + await new Promise((resolve) => { landOlder = resolve; }); + applyUpdate(query, update); + }); + const wall = sinon.stub(Date, 'now'); + try { + wall.returns(1000000); + service.noteEvacuated('someapp'); + wall.returns(2000000); + service.noteEvacuated('otherapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + landOlder(); + await new Promise((resolve) => { setImmediate(resolve); }); + } finally { + wall.restore(); + } + + expect(markerStore.get('residentialDos').lastEvacuationAt, 'an older departure replaced a newer one').to.equal(2000000); + }); + it('gives an empty location list the longest wait, not the shortest', async () => { // `index < 0 ? ordered.length : index` made an EMPTY list position 0 - // the front of the queue - inverting the rule the sibling test asserts. An From e353923bc675ca9141b68253d0253b75d93b5d70 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:26:02 +0100 Subject: [PATCH 186/369] fix(appsync): a peer whose pending registrations this node's database could not store is not counted as synced from A sync completes once enough peers have each answered in full and had all of it recorded. A pending registration the node's database fails to store leaves that peer's answer incomplete, so its request ends and another peer is asked, as for the other three streams. A registration that fails its own checks is the peer's, and is skipped. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxCommunication.js | 15 ++++++++++++ tests/unit/fluxCommunication.test.js | 28 +++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 7452f6614a..2de28294c3 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -4,6 +4,7 @@ const hash = require('object-hash'); const WebSocket = require('ws'); const log = require('../lib/log'); const serviceHelper = require('./serviceHelper'); +const dbHelper = require('./dbHelper'); const messageStore = require('./appMessaging/messageStore'); const verificationHelper = require('./verificationHelper'); const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); @@ -128,14 +129,28 @@ async function handleTempSyncResponse(message, peerSocket) { if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleTempSyncResponse - Received ${messages.length} temp messages from ${peerKey} (done: ${!!done})`); let stored = 0; + let writeFailed = false; for (const msg of messages) { try { const result = await messageStore.storeAppTemporaryMessage(msg, { furtherVerification: true }); if (result === true || result === false) stored += 1; } catch (err) { log.error(`Temp sync message failed: ${err.message}`); + // A message this node's database could not record leaves the answer + // incomplete; one that fails its own checks is the peer's, and is skipped. + if (dbHelper.isDatabaseError(err)) { + writeFailed = true; + break; + } } } + if (writeFailed) { + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleTempSyncResponse - a write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; + } log.info(`handleTempSyncResponse - Processed ${stored} of ${messages.length} messages`); // COUNTED LIKE THE REST. A peer is credited once it has delivered every // stream it was asked for, so the record that admits its responses stays diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index 7afed70a1c..b402df72d0 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -2665,6 +2665,34 @@ describe('fluxCommunication tests', () => { expect(completions).to.deep.equal([{ syncType: 'apprunning', peerKey: PEER }]); }); + describe('a pending registration', () => { + const temp = () => ({ + type: 'fluxappregister', version: 1, signature: 'sig', timestamp: Date.now(), hash: 'h1', appSpecifications: { name: 'a' }, + }); + + it('this node\'s database could not store ends the apptemp request', async () => { + sinon.stub(messageStore, 'storeAppTemporaryMessage').rejects(new MongoNetworkTimeoutError('connection timed out')); + + await respond('handleTempSyncResponse', 'fluxapptempsync', [temp(), temp()], true); + + expect(writeFailures, 'a response the node could not store left the request open').to.deep.equal([PEER]); + expect(completions, 'a response the node could not store was counted').to.deep.equal([]); + sinon.assert.calledOnce(messageStore.storeAppTemporaryMessage); + }); + + it('that fails its own checks is skipped, and the rest of the answer counted', async () => { + sinon.stub(messageStore, 'storeAppTemporaryMessage') + .onFirstCall().rejects(new Error('Invalid Flux App specifications')) + .onSecondCall().resolves(true); + + await respond('handleTempSyncResponse', 'fluxapptempsync', [temp(), temp()], true); + + expect(writeFailures, 'a message the peer got wrong ended the request').to.deep.equal([]); + expect(completions).to.deep.equal([{ syncType: 'apptemp', peerKey: PEER }]); + sinon.assert.calledTwice(messageStore.storeAppTemporaryMessage); + }); + }); + // The request this answer belonged to ended while it was being stored - the // peer reconnected and was asked again - so a failure here ends nothing. it('ends nothing when the request it answered has already ended', async () => { From 2182f290c5cb84f5ebd59a20f9f36aea0f83cbe1 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:35:35 +0100 Subject: [PATCH 187/369] fix(appsync): a hash sync and a registry rebuild never run at once A hash sync storing messages while the registry is unbuilt leaves their rows to the rebuild, which misses any stored after it read the log. Every hash sync and rebuild holds the orchestrator's sync flag: the readiness fallback's rebuild waits out a hash sync and is asked again on the next block, and the operator's check of missing app messages is queued for the orchestrator, which runs it once nothing holds the node. The route answers as before, saying the check is queued. Co-Authored-By: Claude Opus 5.5 --- .../appMessaging/appHashSyncService.js | 4 +- .../appMessaging/appSyncOrchestrator.js | 44 +++++++++++- ZelBack/src/services/utils/appSyncEvents.js | 3 + tests/unit/appHashSyncService.test.js | 18 ++++- tests/unit/appSyncOrchestrator.test.js | 71 +++++++++++++++++++ 5 files changed, 136 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/appMessaging/appHashSyncService.js b/ZelBack/src/services/appMessaging/appHashSyncService.js index e46f2e8e28..0d0860e3b2 100644 --- a/ZelBack/src/services/appMessaging/appHashSyncService.js +++ b/ZelBack/src/services/appMessaging/appHashSyncService.js @@ -846,8 +846,8 @@ async function triggerAppHashesCheckAPI(req, res) { res.json(errMessage); return; } - syncMissingHashes({ force: true }).catch((err) => log.error(`syncMissingHashes API trigger: ${err.message}`)); - const resultsResponse = messageHelper.createSuccessMessage('Running sync on missing application messages'); + appSyncEvents.emit(EVENTS.HASH_CHECK_REQUESTED); + const resultsResponse = messageHelper.createSuccessMessage('Sync on missing application messages queued'); res.json(resultsResponse); } catch (error) { log.error(error); diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index a65162af4c..5f79d7b4bd 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -119,6 +119,11 @@ class AppSyncOrchestrator { #ephemeralWriteFailedHandler = null; #ephemeralProgressHandler = null; #hashUnresolvedHandler = null; + + #hashCheckRequestedHandler = null; + + // An operator's check of every missing app message, waiting for the node. + #hashCheckRequested = false; #broadcastStarted = null; #started = false; #syncInProgress = false; @@ -269,6 +274,12 @@ class AppSyncOrchestrator { this.#hashUnresolvedHandler = () => this.#onHashUnresolved(); appSyncEvents.on(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); + this.#hashCheckRequestedHandler = () => { + this.#hashCheckRequested = true; + this.#runRequestedHashCheck(); + }; + appSyncEvents.on(EVENTS.HASH_CHECK_REQUESTED, this.#hashCheckRequestedHandler); + this.#blockReceivedHandler = (blockHeight) => { this.#onBlocksProcessed(blockHeight); }; @@ -893,6 +904,25 @@ class AppSyncOrchestrator { this.#checkReadiness(); this.#checkHashRetry(blockHeight); } + this.#runRequestedHashCheck(); + } + + /** + * Runs the check an operator asked for, once no hash sync or registry build holds + * the node; a request made while one runs waits for a later block. + * @returns {Promise} + */ + async #runRequestedHashCheck() { + if (!this.#hashCheckRequested || this.#syncInProgress) return; + this.#hashCheckRequested = false; + this.#syncInProgress = true; + try { + await appHashSyncService.syncMissingHashes({ force: true, currentHeight: this.#lastBlockHeight }); + } catch (error) { + log.error(`AppSyncOrchestrator - Requested hash check failed: ${error.message}`); + } finally { + this.#syncInProgress = false; + } } #onHashUnresolved() { @@ -1067,7 +1097,16 @@ class AppSyncOrchestrator { // Block timer fired but hash sync / DB rebuild never completed — rebuild from whatever data we have if (blockTimerExpired && !this.#dbRebuilt) { - await this.#rebuildDb(); + // Never beside a hash sync: one storing messages while the registry is unbuilt + // leaves their rows to this build, which would miss any stored after it read + // the log. A build that has to wait is asked again on the next block. + if (this.#syncInProgress) return; + this.#syncInProgress = true; + try { + await this.#rebuildDb(); + } finally { + this.#syncInProgress = false; + } if (!this.#dbRebuilt) return; } @@ -1227,6 +1266,9 @@ class AppSyncOrchestrator { if (this.#hashUnresolvedHandler) { appSyncEvents.removeListener(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); } + if (this.#hashCheckRequestedHandler) { + appSyncEvents.removeListener(EVENTS.HASH_CHECK_REQUESTED, this.#hashCheckRequestedHandler); + } if (this.#blockReceivedHandler) { this.#blockEmitter.removeListener('blocksProcessed', this.#blockReceivedHandler); } diff --git a/ZelBack/src/services/utils/appSyncEvents.js b/ZelBack/src/services/utils/appSyncEvents.js index 6f3f1a0c21..7df3ce9c8f 100644 --- a/ZelBack/src/services/utils/appSyncEvents.js +++ b/ZelBack/src/services/utils/appSyncEvents.js @@ -12,6 +12,9 @@ const EVENTS = Object.freeze({ READINESS_LOST: 'readinessLost', HASH_RESPONSE_RECEIVED: 'hashResponseReceived', HASH_UNRESOLVED: 'hashUnresolved', + // An operator asked for a check of every missing app message; the orchestrator + // runs it once no hash sync or registry build holds the node. + HASH_CHECK_REQUESTED: 'hashCheckRequested', }); module.exports = { appSyncEvents, EVENTS }; diff --git a/tests/unit/appHashSyncService.test.js b/tests/unit/appHashSyncService.test.js index ee82ef3c14..ac1a85d307 100644 --- a/tests/unit/appHashSyncService.test.js +++ b/tests/unit/appHashSyncService.test.js @@ -5,6 +5,7 @@ const proxyquire = require('proxyquire').noCallThru(); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { Readable } = require('stream'); const config = require('config'); +const { appSyncEvents: syncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); const MESSAGES = config.database.appsglobal.collections.appsMessages; @@ -162,7 +163,19 @@ describe('appHashSyncService tests', () => { describe('triggerAppHashesCheckAPI', () => { - it('should trigger sync when authorized', async () => { + // queued for the sync orchestrator, which runs it once no hash sync or registry + // build holds the node + let requested; + const onRequested = () => { requested += 1; }; + beforeEach(() => { + requested = 0; + syncEvents.on(SYNC_EVENTS.HASH_CHECK_REQUESTED, onRequested); + }); + afterEach(() => { + syncEvents.removeListener(SYNC_EVENTS.HASH_CHECK_REQUESTED, onRequested); + }); + + it('should queue the check when authorized, running nothing itself', async () => { const req = {}; const res = { json: sinon.stub() }; @@ -173,6 +186,8 @@ describe('appHashSyncService tests', () => { expect(res.json.calledOnce).to.be.true; expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req))).to.be.true; + expect(requested, 'the check was not queued').to.equal(1); + sinon.assert.notCalled(dbHelperStub.findInDatabase); }); it('should deny unauthorized access', async () => { @@ -185,6 +200,7 @@ describe('appHashSyncService tests', () => { expect(res.json.calledOnce).to.be.true; expect(messageHelperStub.errUnauthorizedMessage.calledOnce).to.be.true; + expect(requested, 'an unauthorized caller queued a check').to.equal(0); }); it('should handle errors gracefully', async () => { diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index 03aadb04dd..fd5e86ef0e 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -2103,10 +2103,81 @@ describe('AppSyncOrchestrator', () => { blockEmitter.emit('blocksProcessed', 2555000 + i); } await clock.tickAsync(0); + // the burst arrives inside one hash retry, which the build waits out + blockEmitter.emit('blocksProcessed', 2555261); + await clock.tickAsync(0); // The block timer fallback tries rebuildDb again expect(reindexStub.callCount).to.be.greaterThan(1); }); + + // A hash sync storing messages while the registry is unbuilt leaves their rows + // to the build, which would miss any stored after it read the log. + it('never runs the fallback\'s DB rebuild beside a hash sync', async () => { + reindexStub.onFirstCall().rejects(new Error('reindex failed')); + let finishRetry; + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peersUp(); + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + syncMissingHashesStub.callsFake(() => new Promise((resolve) => { finishRetry = resolve; })); + + for (let i = 1; i <= 260; i += 1) { + blockEmitter.emit('blocksProcessed', 2555000 + i); + } + await clock.tickAsync(0); + expect(finishRetry, 'no hash retry was under way, so nothing below was tested').to.be.a('function'); + expect(reindexStub.callCount, 'the DB rebuild ran beside a hash sync').to.equal(1); + + finishRetry({ resolved: 0, missing: 0, unreachable: 0, nextRetryHeight: null }); + await clock.tickAsync(0); + blockEmitter.emit('blocksProcessed', 2555261); + await clock.tickAsync(0); + expect(reindexStub.callCount, 'the DB rebuild was not run once the hash sync ended').to.equal(2); + }); + }); + + // An operator asks for a check of every missing app message. It is queued, and + // runs once nothing else holds the node. + describe('a requested hash check', () => { + it('runs at once when nothing holds the node', async () => { + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peersUp(); + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + syncMissingHashesStub.resetHistory(); + + appSyncEvents.emit(EVENTS.HASH_CHECK_REQUESTED); + await clock.tickAsync(0); + + sinon.assert.calledOnce(syncMissingHashesStub); + expect(syncMissingHashesStub.firstCall.args[0].force).to.equal(true); + }); + + it('waits for a hash sync under way, then runs on a later block', async () => { + let finishSync; + syncMissingHashesStub.callsFake(() => new Promise((resolve) => { finishSync = resolve; })); + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peersUp(); + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + expect(finishSync, 'no hash sync was under way, so nothing below was tested').to.be.a('function'); + + appSyncEvents.emit(EVENTS.HASH_CHECK_REQUESTED); + await clock.tickAsync(0); + expect(syncMissingHashesStub.getCalls().filter((c) => c.args[0].force), 'the check ran beside a hash sync').to.have.length(0); + + syncMissingHashesStub.resolves({ resolved: 0, missing: 0, unreachable: 0, nextRetryHeight: 9999999 }); + finishSync({ resolved: 0, missing: 0, unreachable: 0, nextRetryHeight: 9999999 }); + await clock.tickAsync(0); + blockEmitter.emit('blocksProcessed', 2555001); + await clock.tickAsync(0); + + expect(syncMissingHashesStub.getCalls().filter((c) => c.args[0].force), 'the queued check never ran').to.have.length(1); + }); }); describe('dbReady on fallback paths', () => { From a6dee3e1e3b814c5cfe9d5491d8d339f9f1e85d0 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:37:05 +0100 Subject: [PATCH 188/369] docs(config): the MongoDB floor's patch level is the oldest 6.0 release the fleet runs in numbers Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index eae664a480..f01136e7d0 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -145,7 +145,8 @@ module.exports = { minimumDockerAllowedVersion: '28.1.1', // The MongoDB a node runs: the oldest release FluxOS supports. From 6.0 a // $group or $sort past the memory limit spills to disk rather than failing, - // which the registry rebuild and the duplicate merges at boot rely on. + // which the registry rebuild and the duplicate merges at boot rely on; the + // patch level is the oldest 6.0 release the fleet runs in numbers. minimumMongoDbAllowedVersion: '6.0.27', fluxTeamFluxID: '1hjy4bCYBJr4mny4zCE85J94RXa8W6q37', // A list, so support can be granted to (or revoked from) an identity without From 9eca431abeb283a8b856311d959be3c364dda512 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:51:09 +0100 Subject: [PATCH 189/369] fix(apps): a removal finished after the network was told the app had gone tells it nothing again Co-Authored-By: Claude Opus 5.5 --- .../services/appLifecycle/appUninstaller.js | 16 ++++++--- tests/unit/appUninstaller.test.js | 35 ++++++++++++++++++- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index c502515e01..9545cc3674 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -58,8 +58,9 @@ function setOnComponentRemoved(callback) { onComponentRemoved = callback; } -// Apps whose removal stopped part way, each being run again until it finishes. -const unfinishedRemovals = new Set(); +// Apps whose removal stopped part way, each being run again until it finishes, with +// whether the run that finishes it is still to tell the network. +const unfinishedRemovals = new Map(); /** * Runs the removal of an app that stopped part way again until it finishes. Until @@ -67,12 +68,14 @@ const unfinishedRemovals = new Set(); * announcements leave it out. A call for an app already held does nothing. * @param {string} appName * @param {boolean} force Passed to each removal. - * @param {boolean} announce Whether the removal that finishes tells the network. + * @param {boolean} announce Whether the removal that finishes tells the network; it + * does not once the network has been told meanwhile (announceRemoval). * @returns {void} */ function finishRemoval(appName, force, announce) { if (unfinishedRemovals.has(appName)) return; - unfinishedRemovals.add(appName); + const pending = { announce }; + unfinishedRemovals.set(appName, pending); globalState.departingApps.enter(appName); log.warn(`${appName} was not fully removed; held stopped and unannounced until it is`); // first run once the removal that stopped has released the node's removal lock @@ -83,7 +86,7 @@ function finishRemoval(appName, force, announce) { const holder = globalState.operationHolding(); if (holder) throw new Error(`another ${holder} holds the node`); // eslint-disable-next-line no-use-before-define - const outcome = await removeAppLocally(appName, null, force, true, announce); + const outcome = await removeAppLocally(appName, null, force, true, pending.announce); if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { throw new Error(`${appName} was not removed (${outcome})`); } @@ -121,6 +124,9 @@ async function announceRemoval(appName) { }; log.info('Broadcasting appremoved message to the network'); await fluxCommunicationMessagesSender.broadcastMessageToAll(appRemovedMessage); + // a removal still being finished has nothing left to tell + const pending = unfinishedRemovals.get(appName); + if (pending) pending.announce = false; const { runningAppsCache } = globalState; if (runningAppsCache.has(appName)) { runningAppsCache.delete(appName); diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 1838bb9348..8b118ee41a 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -53,6 +53,7 @@ describe('appUninstaller tests', () => { let globalStateStub; let announceCycle; let getLocalSocketAddressStub; + let broadcastStub; let runCommandStub; beforeEach(() => { @@ -607,6 +608,7 @@ describe('appUninstaller tests', () => { const { departingApps, announceCycle: realAnnounceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {}); announceCycle = realAnnounceCycle; getLocalSocketAddressStub = sinon.stub().resolves(null); + broadcastStub = sinon.stub().resolves(); globalStateStub = { departingApps, announceCycle, @@ -695,7 +697,7 @@ describe('appUninstaller tests', () => { '../fluxCommunicationMessagesSender': { broadcastMessageToOutgoing: sinon.stub().resolves(), broadcastMessageToIncoming: sinon.stub().resolves(), - broadcastMessageToAll: sinon.stub().resolves(), + broadcastMessageToAll: broadcastStub, }, '../appDatabase/registryManager': { availableApps: sinon.stub().resolves([]), @@ -770,6 +772,37 @@ describe('appUninstaller tests', () => { sinon.assert.calledOnce(getLocalSocketAddressStub); }); + // A hard redeploy tells the network once its reinstall fails, and the cleanup + // that stopped part way is finished afterwards: that message is the one sent. + it('finishes a removal without telling the network again when it was told meanwhile', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + getLocalSocketAddressStub.resolves('10.0.0.1:16127'); + + // the reinstall holds the node, so the run that finishes the removal waits + globalStateStub.installationInProgress = true; + await uninstaller.removeAppLocally('testapp', res, true); + await uninstaller.announceRemoval('testapp'); + globalStateStub.installationInProgress = false; + await settle(() => deleteRow.calledTwice && !globalStateStub.departingApps.has('testapp')); + + sinon.assert.calledTwice(deleteRow); + sinon.assert.calledOnce(broadcastStub); + }); + + it('a canary: the same removal told nothing meanwhile tells the network once it finishes', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + getLocalSocketAddressStub.resolves('10.0.0.1:16127'); + + await uninstaller.removeAppLocally('testapp', res, true); + await settle(() => deleteRow.calledTwice && !globalStateStub.departingApps.has('testapp')); + + sinon.assert.calledOnce(broadcastStub); + }); + it('holds an app the network was already told of, and does not tell it twice', async () => { const deleteRow = sinon.stub().resolves(); deleteRow.onFirstCall().rejects(new Error('connection timed out')); From 09e51ff7c73324bf594411f37dc59f7220b4e511 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 11:01:54 +0100 Subject: [PATCH 190/369] refactor(boot): untilSucceeds runs its step again in a loop Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/utils/untilSucceeds.js | 15 ++++---- tests/unit/untilSucceeds.test.js | 38 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 6 deletions(-) create mode 100644 tests/unit/untilSucceeds.test.js diff --git a/ZelBack/src/services/utils/untilSucceeds.js b/ZelBack/src/services/utils/untilSucceeds.js index 5dff67b39c..2d81681dc1 100644 --- a/ZelBack/src/services/utils/untilSucceeds.js +++ b/ZelBack/src/services/utils/untilSucceeds.js @@ -13,12 +13,15 @@ const BOOT_STEP_RETRY_MS = 15 * 1000; * @returns {Promise<*>} What the step resolved. */ async function untilSucceeds(what, step) { - try { - return await step(); - } catch (error) { - log.error(`${what} failed, running it again in ${BOOT_STEP_RETRY_MS / 1000}s: ${error.message}`); - await serviceHelper.delay(BOOT_STEP_RETRY_MS); - return untilSucceeds(what, step); + for (;;) { + try { + // eslint-disable-next-line no-await-in-loop + return await step(); + } catch (error) { + log.error(`${what} failed, running it again in ${BOOT_STEP_RETRY_MS / 1000}s: ${error.message}`); + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(BOOT_STEP_RETRY_MS); + } } } diff --git a/tests/unit/untilSucceeds.test.js b/tests/unit/untilSucceeds.test.js new file mode 100644 index 0000000000..20e0df8e3c --- /dev/null +++ b/tests/unit/untilSucceeds.test.js @@ -0,0 +1,38 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('untilSucceeds', () => { + let delay; + let untilSucceeds; + let BOOT_STEP_RETRY_MS; + + beforeEach(() => { + delay = sinon.stub().resolves(); + ({ untilSucceeds, BOOT_STEP_RETRY_MS } = proxyquire('../../ZelBack/src/services/utils/untilSucceeds', { + '../serviceHelper': { delay }, + '../../lib/log': { error: sinon.stub(), info: sinon.stub(), warn: sinon.stub() }, + })); + }); + + it('answers what the step resolved, without waiting, when it succeeds at once', async () => { + const step = sinon.stub().resolves('done'); + + expect(await untilSucceeds('a step', step)).to.equal('done'); + + sinon.assert.calledOnce(step); + sinon.assert.notCalled(delay); + }); + + it('runs a failing step again after each wait until it succeeds', async () => { + const step = sinon.stub(); + step.onCall(0).rejects(new Error('database did not answer')); + step.onCall(1).rejects(new Error('database did not answer')); + step.onCall(2).resolves('done'); + + expect(await untilSucceeds('a step', step)).to.equal('done'); + + sinon.assert.calledThrice(step); + expect(delay.args).to.deep.equal([[BOOT_STEP_RETRY_MS], [BOOT_STEP_RETRY_MS]]); + }); +}); From 9ab58518fe2698ff0dc68733494770fc8083b63f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 11:02:16 +0100 Subject: [PATCH 191/369] docs(appsync): two comments name what the code does now Co-Authored-By: Claude Opus 5.5 --- .../services/appMessaging/appSyncOrchestrator.js | 16 +++++----------- .../src/services/appMessaging/messageVerifier.js | 2 +- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index 5f79d7b4bd..7a57bc1ab5 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -580,19 +580,13 @@ class AppSyncOrchestrator { this.#reconcile(); } - /** - * A peer sent something this node cannot attribute to it. - * - * Its request ends here rather than on a deadline, because the answer is - * already known: a stream with a hole in it is not a survey, and waiting the - * peer out would spend one of very few slots on an answer that cannot be - * counted whatever else arrives. The record stands, so the peer is not asked - * again on this connection. - * @param {string} peerKey ip:port of the peer whose response failed. - * @returns {void} - */ /** * End a peer's request whose answer cannot be taken, and ask another peer. + * + * Ended here rather than on a deadline: an answer with a hole in it cannot be + * counted whatever else arrives, and waiting the peer out would spend one of very + * few slots on it. The record stands, so the peer is not asked again on this + * connection. * @param {string} peerKey * @param {string} outcome What the request is settled as. * @param {string} why For the log. diff --git a/ZelBack/src/services/appMessaging/messageVerifier.js b/ZelBack/src/services/appMessaging/messageVerifier.js index 90d080096a..c39487e1d9 100644 --- a/ZelBack/src/services/appMessaging/messageVerifier.js +++ b/ZelBack/src/services/appMessaging/messageVerifier.js @@ -870,7 +870,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { // The hash is marked as holding its message only once the message's effect on // the registry is written, at each way out below: a failure before that leaves // the hash unresolved, and resolving it again writes the registry - // (resolveStoredMessage). + // (alignStoredMessagesWithPayments). const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); const daemonHeight = syncStatus.data.height; From c576704a49b1f0628ff09614690f373b12311742 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 11:14:39 +0100 Subject: [PATCH 192/369] fix(id): a login page is never told its phrase is no longer valid while a login is using it A login marks its phrase used, in one conditional update, instead of deleting it: of the logins that try a phrase, one marks it and writes its login. Each is counted from before it tries the phrase until after it has published how it ended. A page waiting on its phrase reads the login, then the phrase: a phrase absent, or used with no login counted against it, is no longer valid; one unused, or used by a login still counted, is waited on. Once no login is counted the page reads again. The active login phrases list leaves used phrases out. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/idService.js | 186 ++++++++++++++--------- tests/unit/idService.test.js | 245 ++++++++++++++++++++++++++++-- 2 files changed, 346 insertions(+), 85 deletions(-) diff --git a/ZelBack/src/services/idService.js b/ZelBack/src/services/idService.js index 9ab85c988f..ab719696cb 100644 --- a/ZelBack/src/services/idService.js +++ b/ZelBack/src/services/idService.js @@ -36,8 +36,28 @@ const LOGIN_PHRASE_VALID_MS = 15 * 60 * 1000; // and the login was not written. The login websocket waits on it. const loginOutcomes = new EventEmitter(); loginOutcomes.setMaxListeners(0); -// The phrases a login has used and not yet published the outcome of. -const loginsInProgress = new Set(); +// For each phrase, how many logins are between trying to use it and ending. A login +// is counted before it tries the phrase and until after it has published how it +// ended, so a used phrase nobody is counted against has ended without a login or +// with one already written. +const loginsInProgress = new Map(); +// A phrase not yet used by a login. +const UNUSED_PHRASE = { usedAt: { $exists: false } }; + +function enterLogin(phrase) { + loginsInProgress.set(phrase, (loginsInProgress.get(phrase) ?? 0) + 1); +} + +function leaveLogin(phrase) { + const held = loginsInProgress.get(phrase); + if (held > 1) { + loginsInProgress.set(phrase, held - 1); + return; + } + loginsInProgress.delete(phrase); + // a page waiting on the phrase reads again: no login is left to publish anything + loginOutcomes.emit(phrase, { ended: true }); +} /** * Publish how a login that used its phrase ended. @@ -45,21 +65,27 @@ const loginsInProgress = new Set(); * @param {{data?: object, error?: object}} outcome */ function publishLoginOutcome(phrase, outcome) { - loginsInProgress.delete(phrase); loginOutcomes.emit(phrase, outcome); } /** - * Removes a login phrase, so that it signs in once. + * Marks a login phrase used, so that it signs in once: of the logins that try it, + * one marks it. The record stays until its TTL removes it, so a login page reads + * a used phrase as used rather than as never issued. * @param {string} phrase - * @returns {Promise} Whether this call removed it. Rejects when the + * @returns {Promise} Whether this call marked it. Rejects when the * database does not answer. */ -async function consumeLoginPhrase(phrase) { +async function claimLoginPhrase(phrase) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); const collection = config.database.local.collections.activeLoginPhrases; - return Boolean(await dbHelper.findOneAndDeleteInDatabase(database, collection, { loginPhrase: phrase }, {})); + return Boolean(await dbHelper.findOneAndUpdateInDatabase( + database, + collection, + { loginPhrase: phrase, ...UNUSED_PHRASE }, + { $set: { usedAt: new Date() } }, + )); } @@ -146,6 +172,7 @@ async function verifyLogin(req, res) { req.on('end', async () => { // set once this request has used the phrase, which no other can then use let usedPhrase = null; + let triedPhrase = null; try { const processedBody = serviceHelper.ensureObject(body); const address = processedBody.zelid || processedBody.address; @@ -182,7 +209,7 @@ async function verifyLogin(req, res) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); const collection = config.database.local.collections.activeLoginPhrases; - const query = { loginPhrase: message }; + const query = { loginPhrase: message, ...UNUSED_PHRASE }; const projection = {}; const result = await dbHelper.findOneInDatabase(database, collection, query, projection); @@ -222,12 +249,13 @@ async function verifyLogin(req, res) { privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } // A phrase signs in once: the login is written only by the request that - // removed it. - if (!await consumeLoginPhrase(message)) { + // marked it used. + triedPhrase = message; + enterLogin(message); + if (!await claimLoginPhrase(message)) { throw new Error('Signed message is no longer valid. Please request a new one.'); } usedPhrase = message; - loginsInProgress.add(message); // The signature is the session's credential. The login websocket hands it // to whoever holds the login phrase, so it is kept apart from the login, // in a collection whose TTL index expires it a minute after the login. It @@ -266,6 +294,8 @@ async function verifyLogin(req, res) { const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); if (usedPhrase) publishLoginOutcome(usedPhrase, { error: errMessage }); res.json(errMessage); + } finally { + if (triedPhrase) leaveLogin(triedPhrase); } }); } @@ -345,7 +375,7 @@ async function activeLoginPhrases(req, res) { const database = db.db(config.database.local.database); const collection = config.database.local.collections.activeLoginPhrases; - const query = {}; + const query = UNUSED_PHRASE; const projection = { projection: { _id: 0, loginPhrase: 1, createdAt: 1, expireAt: 1, @@ -562,6 +592,7 @@ async function logoutAllUsers(req, res) { * Answers a login page waiting on its phrase: with the login once the phrase signs * in, or with why it no longer can. The answer is the outcome verifyLogin * publishes for the phrase; a page that connects after the login reads it instead. + * A page waiting on a used phrase reads again once no login is counted against it. * @param {object} ws Web socket. * @param {string} loginphrase */ @@ -569,6 +600,8 @@ async function wsRespondLoginPhrase(ws, loginphrase) { let settled = false; let expiry = null; const noLongerValid = () => messageHelper.createErrorMessage('Signed message is no longer valid. Please request a new one.'); + const database = dbHelper.databaseConnection().db(config.database.local.database); + const query = { loginPhrase: loginphrase }; function stop() { settled = true; @@ -585,7 +618,75 @@ async function wsRespondLoginPhrase(ws, loginphrase) { log.error(error); } } - function onOutcome({ data, error }) { + + // Reads what the phrase has come to and answers it, or waits for the login that + // is to use it. + async function decide() { + let login; + try { + login = await dbHelper.findOneInDatabase(database, config.database.local.collections.loggedUsers, query, {}); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); + return; + } + if (settled) return; + if (login) { + try { + const loginSignature = await dbHelper.findOneInDatabase(database, config.database.local.collections.loginSignatures, query, {}); + const adminZelid = verificationHelperUtils.nodeOperatorZelid(); + if (!adminZelid) { + throw new Error('Node is still starting and cannot establish privileges yet'); + } + let privilage = PRIVILEGE_RESPONSE.USER; + if (login.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(login.zelid)) { + privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; + } else if (login.zelid === adminZelid) { + privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; + } + finish(messageHelper.createDataMessage({ + message: 'Successfully logged in', + zelid: login.zelid, + loginPhrase: login.loginPhrase, + signature: loginSignature ? loginSignature.signature : undefined, + privilage, + createdAt: login.createdAt, + expireAt: login.expireAt, + }), 4012); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); + } + return; + } + + let phrase; + try { + phrase = await dbHelper.findOneInDatabase(database, config.database.local.collections.activeLoginPhrases, query, {}); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4013); + return; + } + if (settled) return; + // A phrase never issued, or expired, no login will use. A used one with no login + // counted against it ended without the login read above. A login still counted + // publishes how it ends. + if (!phrase || (phrase.usedAt && !loginsInProgress.has(loginphrase))) { + finish(noLongerValid()); + return; + } + if (expiry) return; + const issuedAt = +loginphrase.substring(0, 13); + expiry = setTimeout(() => finish(noLongerValid()), Math.max(issuedAt + LOGIN_PHRASE_VALID_MS - Date.now(), 0)); + if (expiry.unref) expiry.unref(); + } + + function onOutcome({ data, error, ended }) { + if (ended) { + if (!settled) decide(); + return; + } if (data) finish(messageHelper.createDataMessage(data), 4012); else finish(error); } @@ -599,64 +700,7 @@ async function wsRespondLoginPhrase(ws, loginphrase) { }; // before the reads, so a login that finishes while they run is not missed loginOutcomes.on(loginphrase, onOutcome); - - const database = dbHelper.databaseConnection().db(config.database.local.database); - const query = { loginPhrase: loginphrase }; - let login; - try { - login = await dbHelper.findOneInDatabase(database, config.database.local.collections.loggedUsers, query, {}); - } catch (error) { - log.error(error); - finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); - return; - } - if (settled) return; - if (login) { - try { - const loginSignature = await dbHelper.findOneInDatabase(database, config.database.local.collections.loginSignatures, query, {}); - const adminZelid = verificationHelperUtils.nodeOperatorZelid(); - if (!adminZelid) { - throw new Error('Node is still starting and cannot establish privileges yet'); - } - let privilage = PRIVILEGE_RESPONSE.USER; - if (login.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(login.zelid)) { - privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; - } else if (login.zelid === adminZelid) { - privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; - } - finish(messageHelper.createDataMessage({ - message: 'Successfully logged in', - zelid: login.zelid, - loginPhrase: login.loginPhrase, - signature: loginSignature ? loginSignature.signature : undefined, - privilage, - createdAt: login.createdAt, - expireAt: login.expireAt, - }), 4012); - } catch (error) { - log.error(error); - finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); - } - return; - } - - let phrase; - try { - phrase = await dbHelper.findOneInDatabase(database, config.database.local.collections.activeLoginPhrases, query, {}); - } catch (error) { - log.error(error); - finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4013); - return; - } - if (settled) return; - // a login that has used the phrase and not yet ended publishes its outcome - if (!phrase && !loginsInProgress.has(loginphrase)) { - finish(noLongerValid()); - return; - } - const issuedAt = +loginphrase.substring(0, 13); - expiry = setTimeout(() => finish(noLongerValid()), Math.max(issuedAt + LOGIN_PHRASE_VALID_MS - Date.now(), 0)); - if (expiry.unref) expiry.unref(); + await decide(); } /** diff --git a/tests/unit/idService.test.js b/tests/unit/idService.test.js index 69f68e187e..3fc39576f7 100644 --- a/tests/unit/idService.test.js +++ b/tests/unit/idService.test.js @@ -8,6 +8,7 @@ const bitcoinMessage = require('bitcoinjs-message'); const proxyquire = require('proxyquire'); const { PassThrough } = require('stream'); const qs = require('qs'); +const config = require('config'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); @@ -669,7 +670,7 @@ describe('idService tests', () => { loginPhrase: `${timestamp - 10000}11111111111111111111111111111`, }); sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({ loginPhrase: 'removed' }); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').resolves({ loginPhrase: 'used' }); await dbHelper.initiateDB(); dbHelper.databaseConnection(); const req = { @@ -698,13 +699,13 @@ describe('idService tests', () => { sinon.assert.calledOnceWithMatch(res.json, expectedError); }); - const loginWithPhrase = async (removed) => { + const loginWithPhrase = async (claimed) => { bitcoinMessageStub.returns(true); const timestamp = Date.now(); const message = `${timestamp - 300000}11111111111111111111111111111`; sinon.stub(dbHelper, 'findOneInDatabase').resolves({ loginPhrase: message }); const insertStub = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').callsFake(removed); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').callsFake(claimed); await dbHelper.initiateDB(); const mockStream = new PassThrough(); mockStream.push(JSON.stringify({ zelid: '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg', signature: '1234356asdf', message })); @@ -715,14 +716,14 @@ describe('idService tests', () => { return { res, insertStub }; }; - it('writes no login when its phrase cannot be removed', async () => { + it('writes no login when its phrase cannot be marked used', async () => { const { res, insertStub } = await loginWithPhrase(async () => { throw new Error('connection timed out'); }); sinon.assert.notCalled(insertStub); sinon.assert.calledOnceWithMatch(res.json, { status: 'error' }); }); - it('writes no login when another request removed its phrase first', async () => { + it('writes no login when another request used its phrase first', async () => { const { res, insertStub } = await loginWithPhrase(async () => null); sinon.assert.notCalled(insertStub); @@ -735,7 +736,7 @@ describe('idService tests', () => { const timestamp = Date.now(); const message = `${timestamp - 300000}11111111111111111111111111111`; sinon.stub(dbHelper, 'findOneInDatabase').resolves({ loginPhrase: message }); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({ loginPhrase: message }); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').resolves({ loginPhrase: message }); const insertStub = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); await dbHelper.initiateDB(); dbHelper.databaseConnection(); @@ -1291,8 +1292,10 @@ describe('idService tests', () => { const zelid = '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg'; let findOne; let insert; + let claim; let phrase; - let phraseStored; + // the phrase's record: absent, or as issued, or marked used + let phraseRecord; const page = () => ({ send: sinon.stub(), close: sinon.stub() }); const sentMessage = (ws) => qs.parse(ws.send.firstCall.args[0]); @@ -1309,14 +1312,20 @@ describe('idService tests', () => { beforeEach(async () => { sinon.stub(bitcoinMessage, 'verify').returns(true); phrase = `${Date.now() - 60000}11111111111111111111111111111`; - phraseStored = true; + phraseRecord = { loginPhrase: phrase }; findOne = sinon.stub(dbHelper, 'findOneInDatabase'); findOne.withArgs(sinon.match.any, 'loggedusers').resolves(null); - findOne.withArgs(sinon.match.any, 'activeloginphrases').callsFake(async () => (phraseStored ? { loginPhrase: phrase } : null)); + // a login asks for the phrase unused; the page asks for it in any state + findOne.withArgs(sinon.match.any, 'activeloginphrases').callsFake(async (db, coll, query) => { + if (!phraseRecord) return null; + if (query.usedAt && phraseRecord.usedAt) return null; + return phraseRecord; + }); insert = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').callsFake(async () => { - phraseStored = false; - return { loginPhrase: phrase }; + claim = sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').callsFake(async () => { + if (!phraseRecord || phraseRecord.usedAt) return null; + phraseRecord = { ...phraseRecord, usedAt: new Date() }; + return phraseRecord; }); await dbHelper.initiateDB(); }); @@ -1343,7 +1352,7 @@ describe('idService tests', () => { insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); const loggingIn = login(); await serviceHelper.delay(20); - expect(phraseStored, 'the phrase was not used, so this proves nothing').to.equal(false); + expect(phraseRecord.usedAt, 'the phrase was not used, so this proves nothing').to.be.a('date'); const ws = page(); await idService.wsRespondLoginPhrase(ws, phrase); @@ -1395,13 +1404,221 @@ describe('idService tests', () => { }); it('is told its phrase is no longer valid when the phrase is gone and no login used it', async () => { - phraseStored = false; + phraseRecord = null; const ws = page(); await idService.wsRespondLoginPhrase(ws, phrase); expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); }); + + // The phrase is marked used before any login is written, and a login is counted + // from before it tries the phrase: a page never reads a used phrase while the + // login using it is uncounted. + it('waits, and is told the login, when it reads the phrase used while the login that used it is still being written', async () => { + let writeLogin; + insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); + const loggingIn = login(); + await serviceHelper.delay(20); + expect(phraseRecord.usedAt, 'the phrase was not used, so this proves nothing').to.be.a('date'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + writeLogin(true); + await loggingIn; + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('waits while the phrase is being marked used, and is told the login', async () => { + let markUsed; + claim.callsFake(() => new Promise((resolve) => { + markUsed = () => { + phraseRecord = { ...phraseRecord, usedAt: new Date() }; + resolve(phraseRecord); + }; + })); + const loggingIn = login(); + await serviceHelper.delay(20); + expect(markUsed, 'no login was marking the phrase, so this proves nothing').to.be.a('function'); + // the mark has landed in the database, and the login has not yet heard so + phraseRecord = { ...phraseRecord, usedAt: new Date() }; + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + markUsed(); + await loggingIn; + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('is told at once that its phrase is no longer valid when it was used by a login that has ended without writing one', async () => { + insert.withArgs(sinon.match.any, 'loggedusers').rejects(new Error('connection timed out')); + await login(); + expect(phraseRecord.usedAt, 'the phrase was not used, so this proves nothing').to.be.a('date'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + }); + + it('is told the login when it reads the phrase used by a login that has written it and ended', async () => { + await login(); + findOne.withArgs(sinon.match.any, 'loggedusers').resolves({ zelid, loginPhrase: phrase }); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + // Two requests of one phrase: one uses it, the other is refused. The page waits + // on whichever is still counted, and reads again once neither is. + it('is told its phrase is no longer valid once the last login counted against it ends, when none wrote a login', async () => { + const clock = sinon.useFakeTimers({ now: Date.now(), toFake: ['setTimeout', 'clearTimeout', 'Date'], shouldAdvanceTime: true }); + try { + insert.withArgs(sinon.match.any, 'loggedusers').rejects(new Error('connection timed out')); + let refuseSecond; + claim.onSecondCall().callsFake(() => new Promise((resolve) => { refuseSecond = () => resolve(null); })); + // the second reads the phrase while it is still unused + const first = login(); + const second = login(); + await first; + expect(refuseSecond, 'the second login was not under way, so this proves nothing').to.be.a('function'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + refuseSecond(); + await second; + await clock.tickAsync(0); + + expect(sentMessage(ws).data.message, 'the page waited out the phrase instead').to.equal('Signed message is no longer valid. Please request a new one.'); + } finally { + clock.restore(); + } + }); + + it('is told the login, not that its phrase is gone, when a refused login ends while the one that used it is still writing', async () => { + let writeLogin; + insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); + const first = login(); + const second = login(); + await second; + expect(writeLogin, 'the first login was not writing, so this proves nothing').to.be.a('function'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + writeLogin(true); + await first; + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + // The login was written before the page connected, and the last login counted + // against the phrase ends while the page is still reading: the page reads again + // rather than taking the end for a phrase that signed nobody in. + it('is told the login when the last login counted ends while it is still reading', async () => { + let refuseSecond; + claim.onSecondCall().callsFake(() => new Promise((resolve) => { refuseSecond = () => resolve(null); })); + const first = login(); + const second = login(); + await first; + expect(refuseSecond, 'the second login was not under way, so this proves nothing').to.be.a('function'); + let answerRead; + const loginRow = { zelid, loginPhrase: phrase }; + findOne.withArgs(sinon.match.any, 'loggedusers').onFirstCall().callsFake(() => new Promise((resolve) => { answerRead = resolve; })); + findOne.withArgs(sinon.match.any, 'loggedusers').resolves(loginRow); + + const ws = page(); + const connecting = idService.wsRespondLoginPhrase(ws, phrase); + refuseSecond(); + await second; + answerRead(loginRow); + await connecting; + + sinon.assert.calledOnce(ws.send); + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('lists only the phrases no login has used as active', async () => { + const list = sinon.stub(dbHelper, 'findInDatabase').resolves([]); + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + + await idService.activeLoginPhrases({}, generateResponse()); + + expect(list.firstCall.args[2]).to.deep.equal({ usedAt: { $exists: false } }); + }); + }); + + // On the database itself: of the logins that try one phrase at once, one marks it + // used and writes its login; the rest are refused, and the phrase stays, used. + describe('a login phrase on the database', () => { + const zelid = '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg'; + let phrase; + let database; + const { activeLoginPhrases, loggedUsers } = config.database.local.collections; + + async function login() { + const stream = new PassThrough(); + stream.push(JSON.stringify({ zelid, signature: '1234356asdf', message: phrase })); + stream.end(); + const res = generateResponse(); + await idService.verifyLogin(stream, res); + await serviceHelper.delay(100); + return res.json.firstCall.args[0]; + } + + beforeEach(async () => { + sinon.stub(bitcoinMessage, 'verify').returns(true); + await dbHelper.initiateDB(); + database = dbHelper.databaseConnection().db(config.database.local.database); + phrase = `${Date.now() - 60000}${Math.random().toString(36).substring(2, 15)}1111111111111111111`; + await database.collection(activeLoginPhrases).insertOne({ loginPhrase: phrase, createdAt: new Date(), expireAt: new Date(Date.now() + 900000) }); + }); + + afterEach(async () => { + sinon.restore(); + await database.collection(activeLoginPhrases).deleteMany({ loginPhrase: phrase }); + await database.collection(loggedUsers).deleteMany({ loginPhrase: phrase }); + }); + + it('signs in one of two logins that try the phrase at once, and leaves the phrase marked used', async () => { + // both have read the phrase unused before either marks it + const markUsed = dbHelper.findOneAndUpdateInDatabase; + let arrived = 0; + let bothArrived; + const barrier = new Promise((resolve) => { bothArrived = resolve; }); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').callsFake(async (...args) => { + arrived += 1; + if (arrived === 2) bothArrived(); + await barrier; + return markUsed(...args); + }); + + const answers = await Promise.all([login(), login()]); + + expect(arrived, 'the two logins did not both try the phrase, so this proves nothing').to.equal(2); + + expect(answers.map((a) => a.status).sort()).to.deep.equal(['error', 'success']); + expect(answers.find((a) => a.status === 'error').data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + expect(await database.collection(loggedUsers).countDocuments({ loginPhrase: phrase }), 'logins written').to.equal(1); + expect((await database.collection(activeLoginPhrases).findOne({ loginPhrase: phrase }))?.usedAt).to.be.a('date'); + }); + + it('refuses a login of a phrase already used', async () => { + expect((await login()).status).to.equal('success'); + + const again = await login(); + + expect(again.status).to.equal('error'); + expect(await database.collection(loggedUsers).countDocuments({ loginPhrase: phrase })).to.equal(1); + }); }); describe('wsRespondSignature tests', () => { From 3f80c01c5b4907950d944d46cc68c8c7ea8205a5 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:34:19 +0100 Subject: [PATCH 193/369] fix(apps): boot's check of a held app decides only that app, and an uninstall ends the hold Boot holds an app whose location record it cannot read until a background check reads the record and starts or removes the app. Each hold now has its own token, and the full uninstall deletes it with the app's row. The check acts only while its own hold is still in place, re-checked with nothing awaited before it acts. An app uninstalled while held, and an install of the same name after that, are left alone. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 22 +++++-- .../services/appLifecycle/appUninstaller.js | 3 + ZelBack/src/services/utils/globalState.js | 6 +- tests/unit/advancedWorkflows.test.js | 8 +-- tests/unit/appReconciler.test.js | 2 +- tests/unit/appStartupManager.test.js | 63 +++++++++++++++++++ tests/unit/appUninstaller.test.js | 43 ++++++++++++- tests/unit/peerNotification.test.js | 4 +- 8 files changed, 137 insertions(+), 14 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index ba24dfa56e..163a7b7311 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -163,16 +163,27 @@ async function readLocation(appName, localSocketAddr) { * Decides an app held in globalState.unconfirmedApps once its location record can * be read, reading it every BOOT_STEP_RETRY_MS until then. A valid record releases * the app and starts the components boot starts; any other answer removes it, the - * app held until the removal has finished or holds it as departing. + * app held until the removal has finished or holds it as departing. An app whose + * hold has ended by then - uninstalled meanwhile, perhaps installed again since - + * is not this check's to decide, and it does nothing. * @param {string} appName * @param {string[]} componentsToStart * @param {string} localSocketAddr + * @param {symbol} hold The hold this check decides, as boot set it. * @returns {Promise} */ -async function confirmHeldApp(appName, componentsToStart, localSocketAddr) { +async function confirmHeldApp(appName, componentsToStart, localSocketAddr, hold) { + const isHeld = () => globalState.unconfirmedApps.get(appName) === hold; await serviceHelper.delay(BOOT_STEP_RETRY_MS); const valid = await untilSucceeds(`Location check of ${appName}`, () => readLocation(appName, localSocketAddr)); await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.BOOT_BEFORE_LOCATION_DECISION, appName); + // Nothing awaits between this check and the decision below, so the hold cannot + // end in between. + if (!isHeld()) { + log.info(`appStartupManager - App ${appName}: its hold ended before its location record was read, nothing to decide`); + fluxEventBus.publish('boot:appConfirmed', { name: appName, valid, released: true }); + return; + } fluxEventBus.publish('boot:appConfirmed', { name: appName, valid }); if (valid) { log.info(`appStartupManager - App ${appName}: its location record is valid, starting it`); @@ -184,7 +195,7 @@ async function confirmHeldApp(appName, componentsToStart, localSocketAddr) { try { await removeAppBootMayNotStart(appName); } finally { - globalState.unconfirmedApps.delete(appName); + if (isHeld()) globalState.unconfirmedApps.delete(appName); } } /** @@ -303,9 +314,10 @@ async function reconcileAppsOnBoot() { // Whether the network placed it elsewhere while this node was down is not // known, so it is neither started nor removed until it is. log.warn(`appStartupManager - App ${appName}: its location record could not be read, held stopped until it can be`); - globalState.unconfirmedApps.add(appName); + const hold = Symbol(appName); + globalState.unconfirmedApps.set(appName, hold); fluxEventBus.publish('boot:appUnconfirmed', { name: appName }); - confirmHeldApp(appName, componentsToStart, localSocketAddr) + confirmHeldApp(appName, componentsToStart, localSocketAddr, hold) .catch((error) => log.error(`appStartupManager - App ${appName}: the location check stopped: ${error.message}`)); results.appsUnconfirmed.push(appName); // eslint-disable-next-line no-continue diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 9545cc3674..30923b42f8 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -1152,6 +1152,9 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // exactly what stops a teardown that fails part way being read as // tampering. dockerService.clearFluxRemovedContainers(appName); + // Nor is there an app left for boot's location check to decide: an install + // of the same name from here on is a new app. + globalState.unconfirmedApps.delete(appName); const databaseStatus2 = { status: 'Database cleaned', }; diff --git a/ZelBack/src/services/utils/globalState.js b/ZelBack/src/services/utils/globalState.js index ab86792bb5..53bad937be 100644 --- a/ZelBack/src/services/utils/globalState.js +++ b/ZelBack/src/services/utils/globalState.js @@ -129,8 +129,12 @@ const departingApps = { // it was down. Until boot reads the records, an app here is not started, elected, // announced, backed up, restored or torn down to be rebuilt. // +// Keyed by app name; the value is the hold, so the check deciding an app can tell +// its own hold from a later one under the same name. An app's full uninstall ends +// its hold. +// // In-memory deliberately: a restart runs the boot check again. -const unconfirmedApps = new Set(); +const unconfirmedApps = new Map(); // Containers intentionally stopped by FluxOS — crash recovery skips die events for these const stoppingContainers = new Set(); diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 755a42459c..e5d7d9e6fe 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -1046,7 +1046,7 @@ describe('advancedWorkflows tests', () => { it('leaves an app boot has not yet confirmed it holds undecided', async () => { const count = sinon.stub(fluxEventBus, 'count'); - globalState.unconfirmedApps.add(appName); + globalState.unconfirmedApps.set(appName, Symbol(appName)); try { await runPass(); } finally { @@ -5688,7 +5688,7 @@ describe('advancedWorkflows tests', () => { it('refuses an app boot has not yet confirmed it holds', async () => { const res = makeRes(); - globalState.unconfirmedApps.add(appname); + globalState.unconfirmedApps.set(appname, Symbol(appname)); let result; try { result = await advancedWorkflows.appendBackupTask(backupReq(), res); @@ -6108,7 +6108,7 @@ describe('advancedWorkflows tests', () => { it('refuses an app boot has not yet confirmed it holds', async () => { const res = makeRes(); - globalState.unconfirmedApps.add(appname); + globalState.unconfirmedApps.set(appname, Symbol(appname)); let result; try { result = await advancedWorkflows.appendRestoreTask(restoreReq(), res); @@ -8767,7 +8767,7 @@ describe('a redeploy asks what it may rebuild before it takes anything down', () // A rebuilt app starts, and boot has yet to decide whether this one may. it('declines while boot has not confirmed the node still holds the app, and leaves it where it is', async () => { - globalState.unconfirmedApps.add('plainapp'); + globalState.unconfirmedApps.set('plainapp', Symbol('plainapp')); const proceed = await advancedWorkflows.mayTearDownToRebuild({ name: 'plainapp', nodes: [] }, res); diff --git a/tests/unit/appReconciler.test.js b/tests/unit/appReconciler.test.js index 02a5676fe7..0d0c32f3e6 100644 --- a/tests/unit/appReconciler.test.js +++ b/tests/unit/appReconciler.test.js @@ -459,7 +459,7 @@ describe('appReconciler tests', () => { }); it('leaves a component alone while boot has not confirmed its app is still held here', async () => { - stubs.globalState.unconfirmedApps.add('App'); + stubs.globalState.unconfirmedApps.set('App', Symbol('App')); await appReconciler.reconcile('www_App'); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index df06284cb8..46fef3a905 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -429,6 +429,69 @@ describe('appStartupManager tests', () => { expect(appReconcilerStub.enqueue.called).to.equal(false); }); + describe('whose hold ends before its record is read', () => { + // The record answers only when the test says, after it has ended the hold. + const recordAnswersLater = () => { + let answer; + dbHelperStub.findInDatabase.onThirdCall().returns(new Promise((resolve) => { answer = resolve; })); + return (records) => answer(records); + }; + + it('is not removed when it was uninstalled meanwhile and its record reads expired', async () => { + const answer = recordAnswersLater(); + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + globalStateStub.unconfirmedApps.delete('AppA'); + answer([]); + await settle(); + + expect(appUninstallerStub.removeAppLocally.called, 'removed an app installed again after its uninstall').to.equal(false); + expect(appReconcilerStub.enqueue.called).to.equal(false); + }); + + it('is not started when it was uninstalled meanwhile and its record reads valid', async () => { + const answer = recordAnswersLater(); + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + globalStateStub.unconfirmedApps.delete('AppA'); + answer([{ expireAt: futureExpiry() }]); + await settle(); + + expect(appReconcilerStub.enqueue.called, 'started an app on the word of a hold that had ended').to.equal(false); + expect(appUninstallerStub.removeAppLocally.called).to.equal(false); + }); + + it('decides nothing for a later hold under the same name, and leaves that hold in place', async () => { + const answer = recordAnswersLater(); + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + const later = Symbol('AppA'); + globalStateStub.unconfirmedApps.set('AppA', later); + answer([]); + await settle(); + + expect(appUninstallerStub.removeAppLocally.called, 'decided an app another hold now covers').to.equal(false); + expect(globalStateStub.unconfirmedApps.get('AppA'), 'released a hold that was not its own').to.equal(later); + }); + + it('leaves a later hold in place when its own removal ends', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + const later = Symbol('AppA'); + appUninstallerStub.removeAppLocally.callsFake(async () => { + globalStateStub.unconfirmedApps.set('AppA', later); + return 'removed'; + }); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + expect(globalStateStub.unconfirmedApps.get('AppA'), 'released a hold set after its removal began').to.equal(later); + }); + }); + it('has only the components boot starts started, when its g: components are left to the election', async () => { dockerServiceStub.dockerListContainers.resolves([ { Names: ['/fluxweb_MixedApp'], State: 'exited' }, diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 8b118ee41a..54fc6e0a8a 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -605,12 +605,13 @@ describe('appUninstaller tests', () => { // is this suite's alone - evicting the entry instead would hand another // object to every module loaded after it, and globalState is a singleton // whose flags decide whether an operation may start at all. - const { departingApps, announceCycle: realAnnounceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {}); + const { departingApps, unconfirmedApps, announceCycle: realAnnounceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {}); announceCycle = realAnnounceCycle; getLocalSocketAddressStub = sinon.stub().resolves(null); broadcastStub = sinon.stub().resolves(); globalStateStub = { departingApps, + unconfirmedApps, announceCycle, removalInProgress: false, installationInProgress: false, @@ -1004,6 +1005,46 @@ describe('appUninstaller tests', () => { sinon.assert.notCalled(logStub.error); }); + // Boot holds an app whose location record it could not read until a check + // decides it. An uninstall in the meantime leaves that check nothing to + // decide, and an install of the same name afterwards is another app. + describe('the boot hold', () => { + it('ends when the app is uninstalled', async () => { + const uninstaller = buildUninstaller(v2Spec); + globalStateStub.unconfirmedApps.set('testapp', Symbol('testapp')); + + await uninstaller.removeAppLocally('testapp', res, true, true, false); + + expect( + globalStateStub.unconfirmedApps.has('testapp'), + 'still held after its uninstall, so boot\'s check would decide an install that came after it', + ).to.be.false; + }); + + it('holds the app while its teardown runs', async () => { + const uninstaller = buildUninstaller(v2Spec); + globalStateStub.unconfirmedApps.set('testapp', Symbol('testapp')); + let heldDuring = null; + uninstaller.setOnComponentRemoved(() => { + heldDuring = globalStateStub.unconfirmedApps.has('testapp'); + }); + + await uninstaller.removeAppLocally('testapp', res, true, true, false); + + expect(heldDuring, 'released part way through its teardown, so it could be started or announced').to.be.true; + }); + + it('stays when a removal is refused', async () => { + const uninstaller = buildUninstaller(v2Spec); + globalStateStub.unconfirmedApps.set('testapp', Symbol('testapp')); + globalStateStub.removalInProgress = true; + + await uninstaller.removeAppLocally('testapp', res, false, true, false); + + expect(globalStateStub.unconfirmedApps.has('testapp'), 'released by a removal that never ran').to.be.true; + }); + }); + // A node stops claiming an app when it decides to hand it back, not when the // container happens to die. The app stays installed until the removal ends, so // without the mark the announcement built in that window re-creates the diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index 254d379192..b2fa209f47 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -255,7 +255,7 @@ describe('peerNotification tests', () => { { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, ], }); - unconfirmedApps.add('app2'); + unconfirmedApps.set('app2', Symbol('app2')); await peerNotification.checkAndNotifyPeersOfRunningApps(); @@ -267,7 +267,7 @@ describe('peerNotification tests', () => { }); it('announces the app again once boot has confirmed it', async () => { - unconfirmedApps.add('app1'); + unconfirmedApps.set('app1', Symbol('app1')); await peerNotification.checkAndNotifyPeersOfRunningApps(); expect(storeAppRunningMessageStub.called, 'announced while unconfirmed').to.be.false; From b0b2346e8aeb9a5e2045adc4117e9918dcf9fa56 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:34:19 +0100 Subject: [PATCH 194/369] test(harness): 3308, an app uninstalled and installed again while boot's decision is held is left alone Co-Authored-By: Claude Opus 5.5 --- ...308-boot-holds-an-app-it-cannot-confirm.js | 93 ++++++++++++++++++- 1 file changed, 91 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index e50a1a6847..cb6cdeef01 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -46,6 +46,8 @@ import { gComponent, watchGComponents } from '../framework/g-components.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; import { loadSharedConfig, BOOT_DRIFT_MS } from '../framework/coupled-knobs.js'; import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { authenticate } from '../auth.js'; +import { fluxTeamKey } from '../framework/keys.js'; const HOLDER = 0; const NODES = 3; @@ -88,10 +90,14 @@ describe('boot holds an app whose location it cannot read', function () { const mixedApp = `e2ebootmixed${stamp}`; // its record is gone const lapsedApp = `e2ebootlapsed${stamp}`; + // its record is gone, and it is uninstalled and installed again while its decision is held + const againApp = `e2ebootagain${stamp}`; const db = `db_${mixedApp}`; const web = `web_${mixedApp}`; const plain = `${plainApp}_${plainApp}`; const lapsed = `${lapsedApp}_${lapsedApp}`; + const again = `${againApp}_${againApp}`; + let againSpec; let holder; let holderDb; const activeStandby = watchGComponents(() => env, { @@ -182,6 +188,7 @@ describe('boot holds an app whose location it cannot read', function () { await pushTestApp(mixedApp, 'v1', 'bootmixed', { user: APP_UID }); await pushImage(plainApp, 'v1'); await pushImage(lapsedApp, 'v1'); + await pushImage(againApp, 'v1'); const apps = await Promise.all([ buildSeedableApp({ env, @@ -191,12 +198,14 @@ describe('boot holds an app whose location it cannot read', function () { }), buildSeedableApp({ env, name: plainApp, instances: NODES, compose: [plainComponent(plainApp)] }), buildSeedableApp({ env, name: lapsedApp, instances: NODES, compose: [plainComponent(lapsedApp)] }), + buildSeedableApp({ env, name: againApp, instances: NODES, compose: [plainComponent(againApp)] }), ]); + [, , , againSpec] = apps; for (const app of apps) { // eslint-disable-next-line no-await-in-loop await installOnNodes(env, app, [HOLDER]); } - await waitFor(async () => (await Promise.all([db, web, plain, lapsed].map(isUp))).every(Boolean), { + await waitFor(async () => (await Promise.all([db, web, plain, lapsed, again].map(isUp))).every(Boolean), { timeout: 300000, interval: 3000, label: 'every app runs on the holder, the mixed app\'s db as its primary', }); const peers = [...Array(NODES).keys()].filter((i) => i !== HOLDER); @@ -204,7 +213,7 @@ describe('boot holds an app whose location it cannot read', function () { // eslint-disable-next-line no-await-in-loop await installOnNodes(env, app, peers); } - await waitFor(async () => (await Promise.all([mixedApp, plainApp, lapsedApp].map((appName) => installedInstanceIndices(env, appName)))) + await waitFor(async () => (await Promise.all([mixedApp, plainApp, lapsedApp, againApp].map((appName) => installedInstanceIndices(env, appName)))) .every((holders) => holders.length === NODES), { timeout: 300000, interval: 5000, label: 'every app is on every node', }); @@ -405,4 +414,84 @@ describe('boot holds an app whose location it cannot read', function () { expect(heardRemoved(lapsedApp, anchors), 'a peer was told of the removal of an app whose record had expired').to.equal(false); }); }); + + // The decision is held after boot has read the expired record. While it is held + // the app is uninstalled and installed again: the install is a new app, which the + // decision about the old one must leave alone. + describe('an app uninstalled and installed again while its decision is held', function () { + let release; + let failedReads; + let heldAtBoot; + let parkedAtDecision; + let installedAfterUninstall; + let installedAgain; + let decision; + let removedAfterDecision; + + before(async function () { + this.timeout(900000); + step('taking the node down for longer than its records live, its location reads refused'); + let anchors; + ({ anchors, release } = await rebootUnreadable([again], { expire: true })); + const holdable = await holder.holdCheckpoint(BEFORE_DECISION, againApp).then(() => true, () => false); + await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + heldAtBoot = Boolean(await heard('boot:appUnconfirmed', (d) => d.name === againApp, 60000, anchors[HOLDER])); + + step('the database answers: the expired record is read, its decision held'); + const answeredFrom = holder.getLastEventId(); + failedReads = await release(); + release = null; + parkedAtDecision = holdable && Boolean(await heard( + 'checkpoint:held', (d) => d.name === BEFORE_DECISION && d.key === againApp, 120000, answeredFrom, + )); + + step('uninstalling the app and installing it again while its decision is held'); + const auth = await authenticate(holder.url, fluxTeamKey()); + await (await fetch(`${holder.url}/apps/appremove/${againApp}`, { headers: { zelidauth: auth.zelidauth } })).text(); + await waitFor(async () => !(await installedNames()).includes(againApp), { + timeout: 120000, interval: 2000, label: 'the app is uninstalled', + }).catch(() => {}); + installedAfterUninstall = await installedNames(); + await installOnNodes(env, againSpec, [HOLDER]).catch(() => {}); + await waitFor(async () => (await installedNames()).includes(againApp), { + timeout: 180000, interval: 2000, label: 'the app is installed again', + }).catch(() => {}); + installedAgain = await installedNames(); + + step('the decision released'); + const releasedFrom = holder.getLastEventId(); + if (holdable) await holder.releaseCheckpoint(BEFORE_DECISION, againApp); + decision = (await heard('boot:appConfirmed', (d) => d.name === againApp, 60000, releasedFrom))?.data ?? null; + removedAfterDecision = Boolean(await heard('app:removed', (d) => d.name === againApp, 30000, releasedFrom)); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await holder?.releaseAllCheckpoints().catch(() => {}); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('holds the app\'s decision after reading its expired record', function () { + expect(heldAtBoot, 'boot did not hold the app whose location it could not read').to.equal(true); + expect(parkedAtDecision, 'the decision was not held, so the uninstall and install below did not land before it').to.equal(true); + }); + + it('uninstalls the app and installs it again while its decision is held', function () { + expect(installedAfterUninstall, 'the app was not uninstalled').to.not.include(againApp); + expect(installedAgain, 'the app was not installed again').to.include(againApp); + }); + + it('leaves the app installed again alone when the decision about the uninstalled one is released', async function () { + this.timeout(240000); + expect(decision, 'the held decision never ran, so nothing below was tested').to.not.equal(null); + expect(decision.valid, 'the record boot read was not the expired one').to.equal(false); + expect(removedAfterDecision, 'the decision about the uninstalled app removed the app installed after it').to.equal(false); + expect(await installedNames(), 'the app installed again was removed').to.include(againApp); + await waitFor(() => isUp(again), { timeout: 200000, interval: 3000, label: 'the app installed again runs' }); + }); + }); }); From e376b3f9c1f255d80c47dfc90cede2b5e09f3a5c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:45:14 +0100 Subject: [PATCH 195/369] fix(db): an index whose name is taken by another key is dropped and rebuilt ensureIndex finds the index a build conflicts with by the property in conflict: by key when the options differ, by name when the key differs. A named index whose key changes between releases is rebuilt at boot. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/dbHelper.js | 9 ++++++--- tests/unit/dbHelperIndexes.test.js | 20 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index 7cc48b18af..8c1346522e 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -244,8 +244,9 @@ async function dedupeByKey(collection, spec, options = {}) { /** * Assert one index, healing the failures that are recoverable. * - * - a pre-existing index with conflicting OPTIONS (IndexOptionsConflict / - * IndexKeySpecsConflict) is dropped by its real name and recreated; + * - a pre-existing index that conflicts is dropped and recreated: the index on + * the same KEY when its options differ (IndexOptionsConflict), the index of + * the same NAME when its key differs (IndexKeySpecsConflict); * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the * index rather than running degraded without it. Every build goes this way, @@ -296,7 +297,9 @@ async function ensureIndex(collection, spec, options = {}, recover = null) { if (!conflict) throw err; const specKeys = JSON.stringify(spec); const indexes = await collection.listIndexes().toArray(); - const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); + const match = err.codeName === 'IndexKeySpecsConflict' + ? indexes.find((idx) => idx.name === options.name) + : indexes.find((idx) => JSON.stringify(idx.key) === specKeys); if (match?.name) { log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); await collection.dropIndex(match.name); diff --git a/tests/unit/dbHelperIndexes.test.js b/tests/unit/dbHelperIndexes.test.js index 4fa392b22e..0f562076d4 100644 --- a/tests/unit/dbHelperIndexes.test.js +++ b/tests/unit/dbHelperIndexes.test.js @@ -74,6 +74,26 @@ describe('dbHelper ensureIndex', () => { sinon.assert.notCalled(logErrorSpy); }); + it('drops the index of the same name when its key differs, and recreates', async () => { + const conflict = new Error('An existing index has the same name as the requested index but a different key'); + conflict.codeName = 'IndexKeySpecsConflict'; + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(conflict); + createIndex.onSecondCall().resolves('by image'); + const collection = stubCollection(createIndex); + collection.listIndexes.returns({ + toArray: async () => [ + { key: { _id: 1 }, name: '_id_' }, + { key: { repotag: 1 }, name: 'by image' }, + ], + }); + + await ensureIndex(collection, { 'compose.repotag': 1 }, { name: 'by image' }); + + sinon.assert.calledOnceWithExactly(collection.dropIndex, 'by image'); + sinon.assert.calledTwice(createIndex); + }); + // A node holding the old non-unique index of the same key also holds the rows it // never refused: the build after the drop meets them, and repairs them the same way. it('repairs duplicate rows that block the build after a conflicting index is dropped', async () => { From 4c418b7aac431e9f6d50a9d475330163c324bfce Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:47:16 +0100 Subject: [PATCH 196/369] test(harness): 3308 asks for the removal again until the node takes it Co-Authored-By: Claude Opus 5.5 --- .../tests/3308-boot-holds-an-app-it-cannot-confirm.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index cb6cdeef01..78100418c7 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -447,10 +447,13 @@ describe('boot holds an app whose location it cannot read', function () { step('uninstalling the app and installing it again while its decision is held'); const auth = await authenticate(holder.url, fluxTeamKey()); - await (await fetch(`${holder.url}/apps/appremove/${againApp}`, { headers: { zelidauth: auth.zelidauth } })).text(); - await waitFor(async () => !(await installedNames()).includes(againApp), { - timeout: 120000, interval: 2000, label: 'the app is uninstalled', - }).catch(() => {}); + // Asked again until it is gone: the node refuses a removal while another runs, + // and boot is removing the apps whose records expired with this one's. + await waitFor(async () => { + if (!(await installedNames()).includes(againApp)) return true; + await (await fetch(`${holder.url}/apps/appremove/${againApp}`, { headers: { zelidauth: auth.zelidauth } })).text(); + return !(await installedNames()).includes(againApp); + }, { timeout: 120000, interval: 3000, label: 'the app is uninstalled' }).catch(() => {}); installedAfterUninstall = await installedNames(); await installOnNodes(env, againSpec, [HOLDER]).catch(() => {}); await waitFor(async () => (await installedNames()).includes(againApp), { From 68c145899378d61803fd210442aa9fbb5c4eef69 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:58:19 +0100 Subject: [PATCH 197/369] test(harness): an app whose chain records a node already holds is installed on it again without seeding them installSeededOnNodes is installOnNodes without the seed, for an app uninstalled from a node and installed again: its chain records stay, and seeding them a second time is refused. 3308 installs its uninstalled app that way. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/reconciler-suite.js | 7 +++++++ .../tests/3308-boot-holds-an-app-it-cannot-confirm.js | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/framework/reconciler-suite.js b/test-infra/runner/framework/reconciler-suite.js index 8638c02811..75111dd97c 100644 --- a/test-infra/runner/framework/reconciler-suite.js +++ b/test-infra/runner/framework/reconciler-suite.js @@ -120,6 +120,13 @@ export async function seedSpecUpdate(env, updated, indices) { // Returns the indices it installed on. export async function installOnNodes(env, app, indices, { timeout = 120000 } = {}) { await seedGlobalSpec(env, app, indices); + return installSeededOnNodes(env, app, indices, { timeout }); +} + +// installOnNodes for an app whose chain records the nodes already hold: an app +// uninstalled from a node and installed on it again, say. A local uninstall +// leaves those records, and seeding them a second time is refused. +export async function installSeededOnNodes(env, app, indices, { timeout = 120000 } = {}) { const teamKey = fluxTeamKey(); await Promise.all(indices.map(async (i) => { const client = env.clients[i]; diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js index 78100418c7..637371ecfc 100644 --- a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -34,7 +34,9 @@ import { createTestEnv } from '../framework/test-env.js'; import { pushImage, pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { bootAndPeer, installOnNodes, installedInstanceIndices } from '../framework/reconciler-suite.js'; +import { + bootAndPeer, installOnNodes, installSeededOnNodes, installedInstanceIndices, +} from '../framework/reconciler-suite.js'; import { crashFluxos, listAppContainers, releaseFluxos, stopAppContainer, } from '../framework/container.js'; @@ -455,7 +457,7 @@ describe('boot holds an app whose location it cannot read', function () { return !(await installedNames()).includes(againApp); }, { timeout: 120000, interval: 3000, label: 'the app is uninstalled' }).catch(() => {}); installedAfterUninstall = await installedNames(); - await installOnNodes(env, againSpec, [HOLDER]).catch(() => {}); + await installSeededOnNodes(env, againSpec, [HOLDER]).catch((error) => step(`the install did not settle: ${error.message}`)); await waitFor(async () => (await installedNames()).includes(againApp), { timeout: 180000, interval: 2000, label: 'the app is installed again', }).catch(() => {}); From c23538eee43ac5b202000ff1ccd9afae92b3456f Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:02:23 +0100 Subject: [PATCH 198/369] fix(db): a node whose database keeps timing out takes on no new apps Four database round trips that run out within 30 minutes hold the node back from new placements, and the hold lifts 30 minutes after the last one. Running apps are left alone: heavy load also makes calls time out. The count is read from both clients' pool events, which the driver emits for every call whatever code made it: a connection closed by its own socket timeout, or a wait for a pooled connection that ran out. Connections a cleared pool closes after the server went away are not counted. Command monitoring would see the same calls, but it decodes every reply a second time for its success events and slows large cursor reads by over half. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 5 + ZelBack/src/services/databaseHealthService.js | 130 ++++++++++++++ ZelBack/src/services/fluxNetworkHelper.js | 1 + ZelBack/src/services/serviceManager.js | 3 + tests/unit/databaseHealthService.test.js | 162 ++++++++++++++++++ tests/unit/globalconfig/default.js | 5 + 6 files changed, 306 insertions(+) create mode 100644 ZelBack/src/services/databaseHealthService.js create mode 100644 tests/unit/databaseHealthService.test.js diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index 0ec3ae33cd..e375e89551 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -50,6 +50,11 @@ module.exports = { // index builds, the explorer restore's deletes and the registry rebuild. callTimeoutMs: 60 * 1000, maintenanceCallTimeoutMs: 15 * 60 * 1000, + // The database is suspect, and the node takes on no new apps, once + // suspectRoundTrips round trips have run out within suspectWindowMs. It stops + // being suspect suspectWindowMs after the last one. + suspectRoundTrips: 4, + suspectWindowMs: 30 * 60 * 1000, local: { database: `${dbPrefix}zelfluxlocal`, collections: { diff --git a/ZelBack/src/services/databaseHealthService.js b/ZelBack/src/services/databaseHealthService.js new file mode 100644 index 0000000000..736118ccc2 --- /dev/null +++ b/ZelBack/src/services/databaseHealthService.js @@ -0,0 +1,130 @@ +/** + * Hold this node back from new placements while its database is timing out + * FluxOS's calls. + * + * WHAT COUNTS. A round trip that ran out: a pooled connection closed by its own + * socket timeout, or a wait for a pooled connection that ran out. Both are read + * from the clients' pool events, which the driver emits for every call whatever + * code made it. A read is retried once, so one stuck read is two round trips. + * + * WHAT DOES NOT. A connection closed because the server went away. The driver + * clears the pool first, and every connection created before the clear then + * closes with the same reason as a timeout. Connection ids only increase, so + * the newest id at the last clear separates the two. + * + * The hold stops new installs and nothing else. Heavy load also makes calls + * time out, so the apps the node already runs are left alone. + */ + +const config = require('config'); +const log = require('../lib/log'); +const fluxNetworkHelper = require('./fluxNetworkHelper'); + +const OWNER = fluxNetworkHelper.PlacementHoldOwner.DATABASE_SUSPECT; +const SUSPECT_ROUND_TRIPS = config.database.suspectRoundTrips; +const SUSPECT_WINDOW_MS = config.database.suspectWindowMs; + +/** + * When each recent round trip ran out, newest last, never more than + * SUSPECT_ROUND_TRIPS of them: the only question asked of it is whether that + * many fall inside the window. + * @type {number[]} + */ +let timedOut = []; +/** Set while the hold stands; it releases the hold when it fires. */ +let releaseTimer = null; + +function monotonicMs() { + return Number(process.hrtime.bigint() / 1000000n); +} + +/** Every round trip it counted is a full window old by now, so none is left to count. */ +function release() { + releaseTimer = null; + fluxNetworkHelper.clearPlacementHold(OWNER); +} + +/** + * Count one round trip that ran out. The hold is set when SUSPECT_ROUND_TRIPS + * fall within SUSPECT_WINDOW_MS, and released SUSPECT_WINDOW_MS after the last. + * @param {string} what Which client, and how it ran out. + */ +function recordTimeout(what) { + const now = monotonicMs(); + timedOut = [...timedOut, now].filter((at) => now - at < SUSPECT_WINDOW_MS).slice(-SUSPECT_ROUND_TRIPS); + log.warn(`databaseHealth - a database round trip ran out (${what})`); + const held = releaseTimer !== null; + if (!held && timedOut.length < SUSPECT_ROUND_TRIPS) return; + if (!held) { + fluxNetworkHelper.setPlacementHold( + OWNER, + `Database calls are timing out: ${SUSPECT_ROUND_TRIPS} round trips ran out within ${SUSPECT_WINDOW_MS / 60000} minutes`, + ); + } + clearTimeout(releaseTimer); + releaseTimer = setTimeout(release, SUSPECT_WINDOW_MS); +} + +/** The pool events of one client, and which of its closes were timeouts. */ +class PoolTimeouts { + /** + * @param {string} name Names the client in the log. + */ + constructor(name) { + this.name = name; + this.newestCreated = 0; + this.newestBeforeClear = 0; + } + + onCreated(event) { + this.newestCreated = Math.max(this.newestCreated, event.connectionId); + } + + onCleared() { + this.newestBeforeClear = this.newestCreated; + } + + onClosed(event) { + if (event.reason !== 'error' || event.connectionId <= this.newestBeforeClear) return; + recordTimeout(`${this.name} client, connection ${event.connectionId} timed out`); + } + + onCheckOutFailed(event) { + if (event.reason !== 'timeout') return; + recordTimeout(`${this.name} client, no pooled connection within the timeout`); + } +} + +/** + * Count the round trips that run out on this client. + * @param {import('mongodb').MongoClient|null} client + * @param {string} name Names the client in the log. + */ +function watch(client, name) { + if (!client) return; + const pool = new PoolTimeouts(name); + client.on('connectionCreated', pool.onCreated.bind(pool)); + client.on('connectionPoolCleared', pool.onCleared.bind(pool)); + client.on('connectionClosed', pool.onClosed.bind(pool)); + client.on('connectionCheckOutFailed', pool.onCheckOutFailed.bind(pool)); +} + +/** + * @returns {boolean} True while the database holds the node back. + */ +function isSuspect() { + return releaseTimer !== null; +} + +/** Forget every count and release the hold. */ +function stop() { + clearTimeout(releaseTimer); + timedOut = []; + release(); +} + +module.exports = { + watch, + isSuspect, + stop, +}; diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 9050780258..a5e434dfb1 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -123,6 +123,7 @@ const STICKY_DOS_STATE = 100; // a new one. const PlacementHoldOwner = Object.freeze({ RESIDENTIAL_DOS: 'residentialDos', + DATABASE_SUSPECT: 'databaseSuspect', }); // Stops the node taking on NEW apps without declaring it unfit for the ones it diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index ab861a9755..a9cf4ea72b 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -11,6 +11,7 @@ const explorerService = require('./explorerService'); const fluxCommunication = require('./fluxCommunication'); const networkStateService = require('./networkStateService'); const fluxNetworkHelper = require('./fluxNetworkHelper'); +const databaseHealthService = require('./databaseHealthService'); // App modular services - replacing appsService const appInstaller = require('./appLifecycle/appInstaller'); const appUninstaller = require('./appLifecycle/appUninstaller'); @@ -265,6 +266,8 @@ async function startFluxFunctions() { fluxNetworkHelper.checkNodeJsVersionAllowed(); // Hard dependencies — nothing starts until these are confirmed. await dbHelper.waitForMongo(); + databaseHealthService.watch(dbHelper.databaseConnection(), 'main'); + databaseHealthService.watch(dbHelper.maintenanceConnection(), 'maintenance'); // As soon as MongoDB answers: a below-floor node stays up holding a sticky // DOS, so /flux/info names the version found and the version required. fluxNetworkHelper.checkMongoDbVersionAllowed(await dbHelper.getMongoDbVersion()); diff --git a/tests/unit/databaseHealthService.test.js b/tests/unit/databaseHealthService.test.js new file mode 100644 index 0000000000..08ef203bbe --- /dev/null +++ b/tests/unit/databaseHealthService.test.js @@ -0,0 +1,162 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const config = require('config'); +const { EventEmitter } = require('events'); +const { MongoClient } = require('mongodb'); +const log = require('../../ZelBack/src/lib/log'); +const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); +const databaseHealthService = require('../../ZelBack/src/services/databaseHealthService'); +const { requireMongo } = require('./dbTestHelper'); + +const { suspectWindowMs } = config.database; +const MINUTE = 60 * 1000; + +describe('databaseHealthService tests', () => { + let clock; + let client; + + const created = (id) => client.emit('connectionCreated', { connectionId: id }); + const closed = (id, reason = 'error') => client.emit('connectionClosed', { connectionId: id, reason }); + const checkOutFailed = (reason = 'timeout') => client.emit('connectionCheckOutFailed', { reason }); + const timeOut = (n) => { for (let i = 0; i < n; i += 1) closed(100 + i); }; + const held = () => fluxNetworkHelper.isPlacementHeld(); + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'hrtime'] }); + client = new EventEmitter(); + databaseHealthService.watch(client, 'main'); + }); + + afterEach(() => { + databaseHealthService.stop(); + clock.restore(); + sinon.restore(); + }); + + it('holds the node back from new placements on the fourth round trip that runs out', () => { + timeOut(3); + expect(held()).to.equal(false); + + timeOut(1); + + expect(held()).to.equal(true); + expect(databaseHealthService.isSuspect()).to.equal(true); + expect(fluxNetworkHelper.getPlacementHold()).to.include('Database calls are timing out'); + }); + + it('counts a wait for a pooled connection that ran out, and no other checkout failure', () => { + checkOutFailed('poolClosed'); + checkOutFailed('connectionError'); + timeOut(3); + expect(held()).to.equal(false); + + checkOutFailed('timeout'); + + expect(held()).to.equal(true); + }); + + it('does not count a connection closed for any reason but an error', () => { + ['stale', 'idle', 'poolClosed'].forEach((reason) => closed(1, reason)); + timeOut(3); + expect(held()).to.equal(false); + + closed(200); + + expect(held()).to.equal(true); + }); + + it('does not count the connections a cleared pool closes, and counts those created after it', () => { + [1, 2, 3, 4, 5].forEach(created); + client.emit('connectionPoolCleared', {}); + [1, 2, 3, 4, 5].forEach((id) => closed(id)); + expect(held()).to.equal(false); + + [6, 7, 8, 9].forEach(created); + [6, 7, 8].forEach((id) => closed(id)); + expect(held()).to.equal(false); + closed(9); + + expect(held()).to.equal(true); + }); + + it('counts only the round trips inside the window', () => { + timeOut(3); + clock.tick(suspectWindowMs); + + timeOut(1); + + expect(held()).to.equal(false); + }); + + it('releases the hold a window after the last round trip that ran out, and each one moves it', () => { + timeOut(4); + clock.tick(suspectWindowMs - MINUTE); + timeOut(1); + clock.tick(suspectWindowMs - MINUTE); + expect(held()).to.equal(true); + + clock.tick(MINUTE); + + expect(held()).to.equal(false); + expect(databaseHealthService.isSuspect()).to.equal(false); + }); + + it('counts afresh after a release', () => { + timeOut(4); + clock.tick(suspectWindowMs); + expect(held()).to.equal(false); + + timeOut(3); + + expect(held()).to.equal(false); + }); + + it('counts the round trips of every client it watches together', () => { + const maintenance = new EventEmitter(); + databaseHealthService.watch(maintenance, 'maintenance'); + timeOut(2); + maintenance.emit('connectionClosed', { connectionId: 1, reason: 'error' }); + expect(held()).to.equal(false); + + maintenance.emit('connectionClosed', { connectionId: 2, reason: 'error' }); + + expect(held()).to.equal(true); + }); + + describe('against the driver', () => { + let real; + + before(requireMongo); + + beforeEach(async () => { + clock.restore(); + databaseHealthService.stop(); + real = new MongoClient(`mongodb://${config.database.url}:${config.database.port}`, { socketTimeoutMS: 300 }); + await real.connect(); + databaseHealthService.watch(real, 'probe'); + }); + + afterEach(async () => { + await real.close(); + }); + + it('counts one stuck read as two round trips, so two stuck reads hold the node', async () => { + const ranOut = sinon.spy(log, 'warn'); + const collection = real.db(config.database.local.database).collection('databasehealthprobe'); + await collection.insertOne({ probe: 1 }); + const stuckRead = () => collection.findOne({ $where: 'sleep(1000) || true' }).catch((error) => error); + + const first = await stuckRead(); + // the canary: the read did run out, and the driver retried it once + expect(first.name).to.equal('MongoNetworkTimeoutError'); + expect(ranOut.getCalls().filter((call) => /round trip ran out/.test(call.args[0]))).to.have.length(2); + expect(fluxNetworkHelper.isPlacementHeld()).to.equal(false); + + await stuckRead(); + + expect(fluxNetworkHelper.isPlacementHeld()).to.equal(true); + await new Promise((resolve) => { setTimeout(resolve, 1200); }); + await collection.drop(); + }); + }); +}); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 305820e636..befea935b2 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -36,6 +36,11 @@ module.exports = { // index builds, the explorer restore's deletes and the registry rebuild. callTimeoutMs: 60 * 1000, maintenanceCallTimeoutMs: 15 * 60 * 1000, + // The database is suspect, and the node takes on no new apps, once + // suspectRoundTrips round trips have run out within suspectWindowMs. It stops + // being suspect suspectWindowMs after the last one. + suspectRoundTrips: 4, + suspectWindowMs: 30 * 60 * 1000, local: { database: 'zelfluxlocaltest', collections: { From 35bee1b7be803ab59ea49ecc4abb5788f24bb7b5 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:06:12 +0100 Subject: [PATCH 199/369] fix(apps): a dependency thread held in the kernel takes the node out of service Every 5 minutes FluxOS reads the threads of mongod, dockerd, containerd, syncthing, fluxd and fluxbenchd in its own PID namespace. A thread in uninterruptible sleep whose context switch count has not moved never ran between two readings, however far apart they are. One whole interval so holds the node back from new placements; six, 30 minutes, set a sticky DOS. Readings are counted rather than time measured, so a VM its hypervisor paused takes none and cannot read as stalled. An app container running one of the same programs has its own namespace and is not read. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 7 + ZelBack/src/services/fluxNetworkHelper.js | 2 + ZelBack/src/services/hostStallService.js | 177 +++++++++++++++++++++ ZelBack/src/services/serviceManager.js | 3 + tests/unit/globalconfig/default.js | 7 + tests/unit/hostStallService.test.js | 181 ++++++++++++++++++++++ 6 files changed, 377 insertions(+) create mode 100644 ZelBack/src/services/hostStallService.js create mode 100644 tests/unit/hostStallService.test.js diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index e375e89551..df9d4f6c10 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -40,6 +40,13 @@ module.exports = { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIfVQ1HBlOYD61OEzV19gSWPG6yVWvz+TyHXflXIHsPg fluxadm-legacy', ], }, + // A thread of a process FluxOS depends on that stays in the kernel without + // running for one interval holds the node back from new placements, and for + // stuckIntervals intervals takes it out of service until it reboots. + hostStall: { + readIntervalMs: 5 * 60 * 1000, + stuckIntervals: 6, + }, database: { url: '127.0.0.1', port: 27017, diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index a5e434dfb1..16818e2d1b 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -99,6 +99,7 @@ const StickyDosOwner = Object.freeze({ NODEJS_FLOOR: 'nodejsFloor', DOCKER_FLOOR: 'dockerFloor', MONGODB_FLOOR: 'mongodbFloor', + HOST_STALL: 'hostStall', }); // Declares the node unfit for the apps it already runs, and is not moved by the @@ -124,6 +125,7 @@ const STICKY_DOS_STATE = 100; const PlacementHoldOwner = Object.freeze({ RESIDENTIAL_DOS: 'residentialDos', DATABASE_SUSPECT: 'databaseSuspect', + HOST_STALL: 'hostStall', }); // Stops the node taking on NEW apps without declaring it unfit for the ones it diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js new file mode 100644 index 0000000000..468ff4b071 --- /dev/null +++ b/ZelBack/src/services/hostStallService.js @@ -0,0 +1,177 @@ +/** + * Take a node out of service when a thread of one of its dependencies is held + * in the kernel for good. + * + * WHAT COUNTS. A thread in uninterruptible sleep (state D) whose context switch + * count is the same at two readings never ran between them, however far apart + * they are. A thread that stays so for one whole interval holds the node back + * from new placements; for STUCK_INTERVALS intervals, the host is broken, and + * the node goes out of service until it reboots. + * + * Readings are counted rather than time measured: a VM its hypervisor paused + * takes no readings while it is paused, so a pause cannot read as a stall. + * + * WHAT IS READ. The threads of the processes FluxOS depends on, in FluxOS's own + * PID namespace. An app container that runs one of the same programs has its + * own namespace and is not read. + */ + +const fs = require('fs').promises; +const path = require('path'); +const config = require('config'); +const log = require('../lib/log'); +const fluxNetworkHelper = require('./fluxNetworkHelper'); + +const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); +const READ_INTERVAL_MS = config.hostStall.readIntervalMs; +const STUCK_INTERVALS = config.hostStall.stuckIntervals; + +/** + * Every dependency thread in D at the last reading, keyed by process, thread + * and the thread's start time so a reused id is a new thread. + * @type {Map} + */ +let tracked = new Map(); +let readTimer = null; +let procRoot = '/proc'; + +/** + * @param {string} file + * @returns {Promise} The file's text, or null when it is gone. + */ +async function readOrNull(file) { + return fs.readFile(file, 'utf8').catch(() => null); +} + +/** + * One thread's state, start time, context switches and wait point, or null + * when it exited while it was read. + * @param {string} taskDir /proc//task/ + */ +async function readThread(taskDir) { + const [stat, status, wchan] = await Promise.all(['stat', 'status', 'wchan'].map((name) => readOrNull(path.join(taskDir, name)))); + if (stat === null || status === null) return null; + // the command name inside the parentheses may itself hold spaces or ')' + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const switches = status.split('\n') + .filter((line) => line.startsWith('voluntary_ctxt_switches') || line.startsWith('nonvoluntary_ctxt_switches')) + .reduce((sum, line) => sum + Number(line.split(/\s+/)[1]), 0); + return { + state: fields[0], startTime: fields[19], switches, wchan: (wchan ?? '').trim() || 'unknown', + }; +} + +/** + * Every thread of every dependency process in this PID namespace. + * @returns {Promise>} + */ +async function readDependencyThreads() { + const ownNamespace = await fs.readlink(path.join(procRoot, 'self', 'ns', 'pid')).catch(() => null); + const pids = (await fs.readdir(procRoot)).filter((entry) => /^\d+$/.test(entry)); + const threads = []; + // eslint-disable-next-line no-restricted-syntax + for (const pid of pids) { + const pidDir = path.join(procRoot, pid); + // eslint-disable-next-line no-await-in-loop + const comm = (await readOrNull(path.join(pidDir, 'comm')))?.trim(); + // eslint-disable-next-line no-continue + if (!DEPENDENCIES.has(comm)) continue; + // eslint-disable-next-line no-await-in-loop + const namespace = await fs.readlink(path.join(pidDir, 'ns', 'pid')).catch(() => null); + // eslint-disable-next-line no-continue + if (namespace !== ownNamespace) continue; + // eslint-disable-next-line no-await-in-loop + const tids = await fs.readdir(path.join(pidDir, 'task')).catch(() => []); + // eslint-disable-next-line no-restricted-syntax + for (const tid of tids) { + // eslint-disable-next-line no-await-in-loop + const thread = await readThread(path.join(pidDir, 'task', tid)); + if (thread) threads.push({ process: comm, pid, tid, ...thread }); + } + } + return threads; +} + +/** + * Carry each thread's count of intervals stuck from the last reading to this + * one. A thread that ran, or left D, starts again. + * @param {Array} threads One reading. + * @returns {Map} + */ +function carryForward(threads) { + const next = new Map(); + threads.filter((thread) => thread.state === 'D').forEach((thread) => { + const key = `${thread.process}/${thread.pid}/${thread.tid}/${thread.startTime}`; + const before = tracked.get(key); + const intervals = before && before.switches === thread.switches ? before.intervals + 1 : 0; + next.set(key, { + process: thread.process, tid: thread.tid, wchan: thread.wchan, switches: thread.switches, intervals, + }); + }); + return next; +} + +/** + * @param {{process: string, tid: string, wchan: string, intervals: number}} thread + * @returns {string} + */ +function describe(thread) { + const minutes = Math.round((thread.intervals * READ_INTERVAL_MS) / 60000); + return `${thread.process} thread ${thread.tid} has been held in the kernel at ${thread.wchan} for ${minutes} minutes`; +} + +/** Hold or release the node by the longest-stuck thread. */ +function act() { + const longest = [...tracked.values()].reduce((worst, thread) => (!worst || thread.intervals > worst.intervals ? thread : worst), null); + if (!longest || longest.intervals === 0) { + fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL); + return; + } + fluxNetworkHelper.setPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL, describe(longest)); + if (longest.intervals >= STUCK_INTERVALS && !fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, `Host stall: ${describe(longest)}`); + } +} + +/** + * Take one reading and act on it. + * @param {{procRoot?: string}} [options] Where /proc is. + */ +async function readOnce(options = {}) { + procRoot = options.procRoot ?? procRoot; + tracked = carryForward(await readDependencyThreads()); + act(); +} + +async function readAndReschedule() { + try { + await readOnce(); + } catch (error) { + log.error(`hostStall - reading the dependency threads failed: ${error.message}`); + } + if (readTimer !== null) readTimer = setTimeout(readAndReschedule, READ_INTERVAL_MS); +} + +/** + * Read now, and every READ_INTERVAL_MS after each reading ends. + * @param {{procRoot?: string}} [options] Where /proc is. + */ +function start(options = {}) { + if (readTimer !== null) return; + procRoot = options.procRoot ?? procRoot; + readTimer = setTimeout(readAndReschedule, 0); +} + +/** Take no more readings and forget every thread. */ +function stop() { + clearTimeout(readTimer); + readTimer = null; + tracked = new Map(); + procRoot = '/proc'; +} + +module.exports = { + start, + stop, + readOnce, +}; diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index a9cf4ea72b..6133c1baea 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -12,6 +12,7 @@ const fluxCommunication = require('./fluxCommunication'); const networkStateService = require('./networkStateService'); const fluxNetworkHelper = require('./fluxNetworkHelper'); const databaseHealthService = require('./databaseHealthService'); +const hostStallService = require('./hostStallService'); // App modular services - replacing appsService const appInstaller = require('./appLifecycle/appInstaller'); const appUninstaller = require('./appLifecycle/appUninstaller'); @@ -264,6 +265,8 @@ async function startFluxFunctions() { // below-floor node stays up holding a sticky DOS, so /flux/info names the // version found and the version required. fluxNetworkHelper.checkNodeJsVersionAllowed(); + // Before the dependencies are waited for: a stalled one is what this sees. + hostStallService.start(); // Hard dependencies — nothing starts until these are confirmed. await dbHelper.waitForMongo(); databaseHealthService.watch(dbHelper.databaseConnection(), 'main'); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index befea935b2..9085bccb3f 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -26,6 +26,13 @@ module.exports = { fluxadm: { sshAuthorizedKeys: [], }, + // A thread of a process FluxOS depends on that stays in the kernel without + // running for one interval holds the node back from new placements, and for + // stuckIntervals intervals takes it out of service until it reboots. + hostStall: { + readIntervalMs: 5 * 60 * 1000, + stuckIntervals: 6, + }, database: { url: database, port: 27017, diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js new file mode 100644 index 0000000000..13bab6ecfc --- /dev/null +++ b/tests/unit/hostStallService.test.js @@ -0,0 +1,181 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const config = require('config'); +const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); +const hostStallService = require('../../ZelBack/src/services/hostStallService'); + +const { stuckIntervals } = config.hostStall; +const OWN_NAMESPACE = 'pid:[4026531836]'; + +describe('hostStallService tests', () => { + let procRoot; + + /** + * One process in the fake /proc. + * @param {string} pid + * @param {string} comm + * @param {Array<{tid: string, state: string, switches: number, wchan?: string, startTime?: string}>} threads + * @param {string} [namespace] + */ + function processWith(pid, comm, threads, namespace = OWN_NAMESPACE) { + const dir = path.join(procRoot, pid); + fs.rmSync(dir, { recursive: true, force: true }); + fs.mkdirSync(path.join(dir, 'ns'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'comm'), `${comm}\n`); + fs.symlinkSync(namespace, path.join(dir, 'ns', 'pid')); + threads.forEach(({ + tid, state, switches, wchan = 'kvm_async_pf_task_wait_schedule', startTime = '1000', + }) => { + const taskDir = path.join(dir, 'task', tid); + fs.mkdirSync(taskDir, { recursive: true }); + const after = [state, ...Array(18).fill('0'), startTime, '0'].join(' '); + fs.writeFileSync(path.join(taskDir, 'stat'), `${tid} (${comm} worker) ${after}\n`); + fs.writeFileSync(path.join(taskDir, 'status'), `Name:\t${comm}\nvoluntary_ctxt_switches:\t${switches}\nnonvoluntary_ctxt_switches:\t7\n`); + fs.writeFileSync(path.join(taskDir, 'wchan'), wchan); + }); + } + + const stuckMongod = (switches = 40) => processWith('100', 'mongod', [{ tid: '101', state: 'D', switches }, { tid: '102', state: 'S', switches: 5 }]); + const read = () => hostStallService.readOnce({ procRoot }); + async function readTimes(n) { + for (let i = 0; i < n; i += 1) await read(); // eslint-disable-line no-await-in-loop + } + const placementHeld = () => fluxNetworkHelper.isPlacementHeld(); + const outOfService = () => fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + + beforeEach(() => { + procRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-')); + fs.mkdirSync(path.join(procRoot, 'self', 'ns'), { recursive: true }); + fs.symlinkSync(OWN_NAMESPACE, path.join(procRoot, 'self', 'ns', 'pid')); + }); + + afterEach(() => { + hostStallService.stop(); + fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + fs.rmSync(procRoot, { recursive: true, force: true }); + sinon.restore(); + }); + + it('holds the node back once a thread stays in D without running for one interval', async () => { + stuckMongod(); + await read(); + expect(placementHeld()).to.equal(false); + + await read(); + + expect(placementHeld()).to.equal(true); + expect(fluxNetworkHelper.getPlacementHold()).to.include('mongod thread 101').and.include('kvm_async_pf_task_wait_schedule'); + expect(outOfService()).to.equal(false); + }); + + it('takes the node out of service once the thread has not run for the configured intervals', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + expect(outOfService()).to.equal(false); + + await read(); + + expect(outOfService()).to.equal(true); + expect(fluxNetworkHelper.isNodeDos()).to.equal(true); + expect(fluxNetworkHelper.getDosMessage()).to.include('Host stall: mongod thread 101'); + }); + + it('keeps the reason it went out of service with while the thread stays stuck', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + const reason = fluxNetworkHelper.getDosMessage(); + + await readTimes(3); + + expect(fluxNetworkHelper.getDosMessage()).to.equal(reason); + }); + + it('starts the count again for a thread that ran between readings, though it is in D again', async () => { + stuckMongod(40); + await readTimes(stuckIntervals); + stuckMongod(41); + await read(); + expect(placementHeld()).to.equal(false); + + await readTimes(stuckIntervals - 1); + expect(outOfService()).to.equal(false); + + await read(); + + expect(outOfService()).to.equal(true); + }); + + it('starts the count again for a thread that left D, and releases the hold', async () => { + stuckMongod(); + await readTimes(3); + expect(placementHeld()).to.equal(true); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 40 }]); + + await read(); + + expect(placementHeld()).to.equal(false); + }); + + it('counts a reused thread id as a new thread', async () => { + stuckMongod(); + await read(); + processWith('100', 'mongod', [{ + tid: '101', state: 'D', switches: 40, startTime: '2000', + }]); + + await read(); + + expect(placementHeld()).to.equal(false); + }); + + it('reads every dependency, and no other program', async () => { + processWith('200', 'nginx', [{ tid: '201', state: 'D', switches: 3 }]); + processWith('300', 'kworker/0:1', [{ tid: '301', state: 'D', switches: 3 }]); + await readTimes(3); + expect(placementHeld()).to.equal(false); + + ['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd'].forEach((comm, i) => { + fs.rmSync(path.join(procRoot, '100'), { recursive: true, force: true }); + processWith('100', comm, [{ tid: `10${i}`, state: 'D', switches: 3 }]); + }); + await readTimes(2); + + expect(fluxNetworkHelper.getPlacementHold()).to.include('fluxbenchd thread 105'); + }); + + it('does not read a dependency running in an app container', async () => { + processWith('400', 'mongod', [{ tid: '401', state: 'D', switches: 3 }], 'pid:[4026532999]'); + + await readTimes(stuckIntervals + 1); + + expect(placementHeld()).to.equal(false); + expect(outOfService()).to.equal(false); + }); + + it('keeps the node out of service when the thread runs again', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 99 }]); + + await read(); + + expect(outOfService()).to.equal(true); + }); + + it('reads a thread whose command name holds spaces and parentheses', async () => { + processWith('500', 'syncthing', []); + const taskDir = path.join(procRoot, '500', 'task', '501'); + fs.mkdirSync(taskDir, { recursive: true }); + fs.writeFileSync(path.join(taskDir, 'stat'), `501 (a) b (c) D ${Array(18).fill('0').join(' ')} 1000 0\n`); + fs.writeFileSync(path.join(taskDir, 'status'), 'voluntary_ctxt_switches:\t3\nnonvoluntary_ctxt_switches:\t1\n'); + fs.writeFileSync(path.join(taskDir, 'wchan'), 'folio_wait_bit'); + + await readTimes(2); + + expect(fluxNetworkHelper.getPlacementHold()).to.include('syncthing thread 501').and.include('folio_wait_bit'); + }); +}); From e9deba5a5107ff9608017a9f2eda0e8e98a4dbdf Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:08:27 +0100 Subject: [PATCH 200/369] fix(apps): a host stall keeps the node out of service until it reboots The trip is written to config/hostStall.json with the kernel's boot id, staged and renamed into place. A start in the same boot restores it before boot acts; a start after a reboot discards it. Without a readable boot id nothing is written and nothing restored, since a restart could not be told from a reboot. A file rather than a database row, because the database may be what is stuck. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 + ZelBack/src/services/hostStallService.js | 98 ++++++++++++++++++++---- ZelBack/src/services/serviceManager.js | 3 +- tests/unit/hostStallService.test.js | 72 ++++++++++++++++- 4 files changed, 159 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 863ad99093..8e3a4c2ec4 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ yarn-error.log* /test/e2e/reports/ selenium-debug.log config/userconfig.js +config/hostStall.json +config/hostStall.json.tmp ZelApps appvolumes .nyc_output/ diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 468ff4b071..f26dd24ddf 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -11,6 +11,11 @@ * Readings are counted rather than time measured: a VM its hypervisor paused * takes no readings while it is paused, so a pause cannot read as a stall. * + * Out of service lasts until the machine reboots, through any number of FluxOS + * restarts: the trip is recorded with the kernel's boot id, and a start in the + * same boot restores it. The record is a file, not a database row, because the + * database may be what is stuck. + * * WHAT IS READ. The threads of the processes FluxOS depends on, in FluxOS's own * PID namespace. An app container that runs one of the same programs has its * own namespace and is not read. @@ -21,6 +26,7 @@ const path = require('path'); const config = require('config'); const log = require('../lib/log'); const fluxNetworkHelper = require('./fluxNetworkHelper'); +const { fluxDirPath } = require('./utils/appConstants'); const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); const READ_INTERVAL_MS = config.hostStall.readIntervalMs; @@ -33,7 +39,20 @@ const STUCK_INTERVALS = config.hostStall.stuckIntervals; */ let tracked = new Map(); let readTimer = null; -let procRoot = '/proc'; +const DEFAULT_PATHS = { + procRoot: '/proc', + recordFile: path.join(fluxDirPath, 'config', 'hostStall.json'), + bootIdPath: config.system.bootIdPath, +}; +let paths = { ...DEFAULT_PATHS }; + +/** + * @param {{procRoot?: string, recordFile?: string, bootIdPath?: string}} options + * Where /proc, the trip record and the boot id are; each one given replaces it. + */ +function usePaths(options) { + paths = { ...paths, ...options }; +} /** * @param {string} file @@ -66,12 +85,12 @@ async function readThread(taskDir) { * @returns {Promise>} */ async function readDependencyThreads() { - const ownNamespace = await fs.readlink(path.join(procRoot, 'self', 'ns', 'pid')).catch(() => null); - const pids = (await fs.readdir(procRoot)).filter((entry) => /^\d+$/.test(entry)); + const ownNamespace = await fs.readlink(path.join(paths.procRoot, 'self', 'ns', 'pid')).catch(() => null); + const pids = (await fs.readdir(paths.procRoot)).filter((entry) => /^\d+$/.test(entry)); const threads = []; // eslint-disable-next-line no-restricted-syntax for (const pid of pids) { - const pidDir = path.join(procRoot, pid); + const pidDir = path.join(paths.procRoot, pid); // eslint-disable-next-line no-await-in-loop const comm = (await readOrNull(path.join(pidDir, 'comm')))?.trim(); // eslint-disable-next-line no-continue @@ -120,8 +139,54 @@ function describe(thread) { return `${thread.process} thread ${thread.tid} has been held in the kernel at ${thread.wchan} for ${minutes} minutes`; } +/** + * @returns {Promise} This boot's id, or null when it cannot be read. + */ +async function currentBootId() { + const id = (await readOrNull(paths.bootIdPath))?.trim(); + return id || null; +} + +/** + * Record the trip, so a FluxOS restart in this boot restores it. + * @param {string} reason + */ +async function recordTrip(reason) { + try { + const bootId = await currentBootId(); + if (!bootId) { + log.warn('hostStall - the boot id cannot be read; the trip lasts only as long as this process'); + return; + } + const staged = `${paths.recordFile}.tmp`; + await fs.writeFile(staged, JSON.stringify({ bootId, reason })); + await fs.rename(staged, paths.recordFile); + } catch (error) { + log.error(`hostStall - the trip could not be recorded: ${error.message}`); + } +} + +/** Restore a trip recorded in this boot, and discard one from an earlier boot. */ +async function restoreTrip() { + try { + const text = await readOrNull(paths.recordFile); + if (text === null) return; + const bootId = await currentBootId(); + if (!bootId) return; + const { bootId: recorded, reason } = JSON.parse(text); + if (recorded !== bootId) { + await fs.rm(paths.recordFile, { force: true }); + log.info('hostStall - the recorded trip is from an earlier boot and is discarded'); + return; + } + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); + } catch (error) { + log.error(`hostStall - the recorded trip could not be read: ${error.message}`); + } +} + /** Hold or release the node by the longest-stuck thread. */ -function act() { +async function act() { const longest = [...tracked.values()].reduce((worst, thread) => (!worst || thread.intervals > worst.intervals ? thread : worst), null); if (!longest || longest.intervals === 0) { fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL); @@ -129,18 +194,20 @@ function act() { } fluxNetworkHelper.setPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL, describe(longest)); if (longest.intervals >= STUCK_INTERVALS && !fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { - fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, `Host stall: ${describe(longest)}`); + const reason = `Host stall: ${describe(longest)}`; + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); + await recordTrip(reason); } } /** * Take one reading and act on it. - * @param {{procRoot?: string}} [options] Where /proc is. + * @param {object} [options] Paths, as usePaths takes them. */ async function readOnce(options = {}) { - procRoot = options.procRoot ?? procRoot; + usePaths(options); tracked = carryForward(await readDependencyThreads()); - act(); + await act(); } async function readAndReschedule() { @@ -153,13 +220,16 @@ async function readAndReschedule() { } /** - * Read now, and every READ_INTERVAL_MS after each reading ends. - * @param {{procRoot?: string}} [options] Where /proc is. + * Restore a trip recorded in this boot, then read now, and every + * READ_INTERVAL_MS after each reading ends. + * @param {object} [options] Paths, as usePaths takes them. + * @returns {Promise} Resolves once a recorded trip is restored. */ -function start(options = {}) { +async function start(options = {}) { if (readTimer !== null) return; - procRoot = options.procRoot ?? procRoot; + usePaths(options); readTimer = setTimeout(readAndReschedule, 0); + await restoreTrip(); } /** Take no more readings and forget every thread. */ @@ -167,7 +237,7 @@ function stop() { clearTimeout(readTimer); readTimer = null; tracked = new Map(); - procRoot = '/proc'; + paths = { ...DEFAULT_PATHS }; } module.exports = { diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 6133c1baea..103566537f 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -266,7 +266,8 @@ async function startFluxFunctions() { // version found and the version required. fluxNetworkHelper.checkNodeJsVersionAllowed(); // Before the dependencies are waited for: a stalled one is what this sees. - hostStallService.start(); + // Awaited so a trip recorded in this boot holds the node before boot acts. + await hostStallService.start(); // Hard dependencies — nothing starts until these are confirmed. await dbHelper.waitForMongo(); databaseHealthService.watch(dbHelper.databaseConnection(), 'main'); diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 13bab6ecfc..aeb1848b90 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -12,6 +12,7 @@ const OWN_NAMESPACE = 'pid:[4026531836]'; describe('hostStallService tests', () => { let procRoot; + let paths; /** * One process in the fake /proc. @@ -39,7 +40,9 @@ describe('hostStallService tests', () => { } const stuckMongod = (switches = 40) => processWith('100', 'mongod', [{ tid: '101', state: 'D', switches }, { tid: '102', state: 'S', switches: 5 }]); - const read = () => hostStallService.readOnce({ procRoot }); + const read = () => hostStallService.readOnce(paths); + const bootIs = (id) => fs.writeFileSync(paths.bootIdPath, `${id}\n`); + const record = () => JSON.parse(fs.readFileSync(paths.recordFile, 'utf8')); async function readTimes(n) { for (let i = 0; i < n; i += 1) await read(); // eslint-disable-line no-await-in-loop } @@ -50,6 +53,8 @@ describe('hostStallService tests', () => { procRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-')); fs.mkdirSync(path.join(procRoot, 'self', 'ns'), { recursive: true }); fs.symlinkSync(OWN_NAMESPACE, path.join(procRoot, 'self', 'ns', 'pid')); + paths = { procRoot, recordFile: path.join(procRoot, 'hostStall.json'), bootIdPath: path.join(procRoot, 'boot_id') }; + bootIs('boot-one'); }); afterEach(() => { @@ -166,6 +171,71 @@ describe('hostStallService tests', () => { expect(outOfService()).to.equal(true); }); + describe('the trip record', () => { + async function trip() { + stuckMongod(); + await readTimes(stuckIntervals + 1); + } + async function restart() { + hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + await hostStallService.start(paths); + } + + it('records the trip with this boot\'s id', async () => { + await trip(); + + expect(record()).to.deep.equal({ bootId: 'boot-one', reason: fluxNetworkHelper.getDosMessage() }); + }); + + it('restores the trip at a start in the same boot', async () => { + await trip(); + const reason = fluxNetworkHelper.getDosMessage(); + + await restart(); + + expect(outOfService()).to.equal(true); + expect(fluxNetworkHelper.getDosMessage()).to.equal(reason); + }); + + it('discards a trip from an earlier boot, and stays in service', async () => { + await trip(); + bootIs('boot-two'); + + await restart(); + + expect(outOfService()).to.equal(false); + expect(fs.existsSync(paths.recordFile)).to.equal(false); + }); + + it('records nothing without a boot id, and is still out of service', async () => { + fs.rmSync(paths.bootIdPath); + + await trip(); + + expect(outOfService()).to.equal(true); + expect(fs.existsSync(paths.recordFile)).to.equal(false); + }); + + it('restores nothing and discards nothing at a start without a boot id', async () => { + await trip(); + fs.rmSync(paths.bootIdPath); + + await restart(); + + expect(outOfService()).to.equal(false); + expect(fs.existsSync(paths.recordFile)).to.equal(true); + }); + + it('starts in service when the record cannot be read', async () => { + fs.writeFileSync(paths.recordFile, '{ not json'); + + await restart(); + + expect(outOfService()).to.equal(false); + }); + }); + it('reads a thread whose command name holds spaces and parentheses', async () => { processWith('500', 'syncthing', []); const taskDir = path.join(procRoot, '500', 'task', '501'); From 36e3256b3838d59a9951bd86edf5d8c880d083b4 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:15:42 +0100 Subject: [PATCH 201/369] fix(apps): every app's containers live in their own slice under flux.slice Under docker's systemd cgroup driver each app container is created with the app's slice as its cgroup parent, flux-.slice with each hyphen escaped the way systemd-escape does, so no app's slice nests inside another's. The app's processes are then found from its name alone, and the apps together share the machine against the system services as one group. Removing a whole app stops its slice once nothing lives in it, and boot stops every empty app slice before it starts any container. A container created before this keeps its parent until its app is next created again. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appSlice.js | 108 ++++++++++++++++++ .../appLifecycle/appStartupManager.js | 3 + .../services/appLifecycle/appUninstaller.js | 2 + ZelBack/src/services/dockerService.js | 7 ++ tests/unit/appSlice.test.js | 98 ++++++++++++++++ tests/unit/appStartupManager.test.js | 22 ++++ tests/unit/appUninstaller.test.js | 23 ++++ tests/unit/dockerService.test.js | 24 ++++ 8 files changed, 287 insertions(+) create mode 100644 ZelBack/src/services/appLifecycle/appSlice.js create mode 100644 tests/unit/appSlice.test.js diff --git a/ZelBack/src/services/appLifecycle/appSlice.js b/ZelBack/src/services/appLifecycle/appSlice.js new file mode 100644 index 0000000000..c605588977 --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appSlice.js @@ -0,0 +1,108 @@ +/** + * Every app's containers live in their own systemd slice, under flux.slice, so + * the app's processes are found and killed from its name alone, without docker + * or the database. Under contention the apps together share the machine + * against the system services as one group, so they cannot starve FluxOS's own + * dependencies. + * + * A slice outlives its last container until it is stopped. Removing an app + * stops its slice, and boot stops every slice left empty. + */ + +const fs = require('fs').promises; +const path = require('path'); +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); + +const PARENT = 'flux.slice'; +const APP_NAME = /^[A-Za-z0-9-]+$/; + +let cgroupRoot = '/sys/fs/cgroup'; + +/** + * @param {string} root Where the cgroup filesystem is mounted. + */ +function useCgroupRoot(root) { + cgroupRoot = root; +} + +/** + * The app's slice unit. systemd reads each '-' in a slice name as one level of + * nesting, so the name escapes it the way systemd-escape does: unescaped, + * `my-app` would land inside `my`'s slice. + * @param {string} appName + * @returns {string|null} Null for a name no app can have. + */ +function sliceName(appName) { + if (typeof appName !== 'string' || !APP_NAME.test(appName)) return null; + return `flux-${appName.replace(/-/g, '\\x2d')}.slice`; +} + +/** + * @param {string} unit A slice under flux.slice. + * @returns {string} Its cgroup directory. + */ +function cgroupOf(unit) { + return path.join(cgroupRoot, PARENT, unit); +} + +/** + * @param {string} unit + * @returns {Promise} Whether any process lives in the slice, or + * null when the slice does not exist. + */ +async function isPopulated(unit) { + const events = await fs.readFile(path.join(cgroupOf(unit), 'cgroup.events'), 'utf8').catch(() => null); + if (events === null) return null; + return /^populated 1$/m.test(events); +} + +/** + * Stop a slice that holds no process. A slice in use is left alone. + * @param {string} unit + * @returns {Promise} True when the slice was stopped. + */ +async function stopIfEmpty(unit) { + if (await isPopulated(unit) !== false) return false; + const { error } = await serviceHelper.runCommand('systemctl', { runAsRoot: true, params: ['stop', unit] }); + if (error) { + log.error(`appSlice - ${unit} could not be stopped: ${error.message}`); + return false; + } + return true; +} + +/** + * Stop the app's slice, once its containers are gone. + * @param {string} appName + */ +async function release(appName) { + const unit = sliceName(appName); + if (unit) await stopIfEmpty(unit); +} + +/** + * Stop every app slice that holds no process: those whose app was removed + * while FluxOS was not running to stop them. + * @returns {Promise} How many were stopped. + */ +async function stopEmptySlices() { + const entries = await fs.readdir(path.join(cgroupRoot, PARENT)).catch(() => []); + const slices = entries.filter((entry) => entry.startsWith('flux-') && entry.endsWith('.slice')); + let stopped = 0; + // eslint-disable-next-line no-restricted-syntax + for (const unit of slices) { + // eslint-disable-next-line no-await-in-loop + if (await stopIfEmpty(unit)) stopped += 1; + } + if (stopped) log.info(`appSlice - stopped ${stopped} empty app slice(s)`); + return stopped; +} + +module.exports = { + sliceName, + cgroupOf, + release, + stopEmptySlices, + useCgroupRoot, +}; diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 163a7b7311..225217ad7a 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -16,6 +16,7 @@ const registryManager = require('../appDatabase/registryManager'); const appReconciler = require('../appMonitoring/appReconciler'); const appUninstaller = require('./appUninstaller'); const appNetworkLinker = require('./appNetworkLinker'); +const appSlice = require('./appSlice'); const portManager = require('../appNetwork/portManager'); const globalState = require('../utils/globalState'); const fluxEventBus = require('../utils/fluxEventBus'); @@ -391,6 +392,8 @@ async function removeAllApps(reason) { async function manageAppsOnBoot(bootContext) { try { + // ahead of every container start, so no slice in use can be empty here + await appSlice.stopEmptySlices(); if (bootContext.firstBoot) { log.info('appStartupManager - First boot (no heartbeat history), waiting for sync'); } else if (bootContext.unreadable) { diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 30923b42f8..cfbdb7e90b 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -23,6 +23,7 @@ const { stopAppMonitoring } = require('../appManagement/appInspector'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); const volumeService = require('../utils/volumeService'); const fluxEventBus = require('../utils/fluxEventBus'); +const appSlice = require('./appSlice'); const { Privilege, authOf } = require('../utils/privileges'); const { RemovalOutcome } = require('../utils/removalOutcome'); const { untilSucceeds, BOOT_STEP_RETRY_MS } = require('../utils/untilSucceeds'); @@ -1044,6 +1045,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen } else { await hardUninstallApplication(appName, appId, appSpecifications, res, stopAppMonitoring, force); } + if (!isComponent) await appSlice.release(appName); // clear node-local runtime state (operator stop lock, crash backoff) for the // removed component(s) so a later reinstall starts from a clean slate diff --git a/ZelBack/src/services/dockerService.js b/ZelBack/src/services/dockerService.js index 46ebdcd434..840f49864b 100644 --- a/ZelBack/src/services/dockerService.js +++ b/ZelBack/src/services/dockerService.js @@ -14,6 +14,7 @@ const { isFluxStorageUrl, storageLinkOf } = require('./utils/fluxStorage'); const log = require('../lib/log'); const cpuBurstHelper = require('./utils/cpuBurstHelper'); const LogFrameDecoder = require('./utils/logFrameDecoder'); +const appSlice = require('./appLifecycle/appSlice'); const globalState = require('./utils/globalState'); @@ -1239,6 +1240,12 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS // eslint-disable-next-line no-use-before-define const dockerInfoResp = await dockerInfo(); log.info(dockerInfoResp); + // The app's own slice is a systemd unit, so it is the parent only under the + // systemd cgroup driver; under another the container keeps docker's default. + const slice = appSlice.sliceName(appName); + if (dockerInfoResp.CgroupDriver === 'systemd' && slice) { + options.HostConfig.CgroupParent = slice; + } const driverStatus = dockerInfoResp.DriverStatus; const backingFs = driverStatus.find((status) => status[0] === 'Backing Filesystem'); // d_type must be true for overlay, docker would not work if not if (backingFs && backingFs[1] === 'xfs') { diff --git a/tests/unit/appSlice.test.js b/tests/unit/appSlice.test.js new file mode 100644 index 0000000000..510437b878 --- /dev/null +++ b/tests/unit/appSlice.test.js @@ -0,0 +1,98 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const appSlice = require('../../ZelBack/src/services/appLifecycle/appSlice'); + +describe('appSlice tests', () => { + let root; + let runCommand; + + function slice(unit, populated) { + const dir = path.join(root, 'flux.slice', unit); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'cgroup.events'), `populated ${populated ? 1 : 0}\nfrozen 0\n`); + } + const stopped = () => runCommand.getCalls().map((call) => call.args[1].params[1]); + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'appslice-')); + appSlice.useCgroupRoot(root); + runCommand = sinon.stub(serviceHelper, 'runCommand').resolves({ error: null }); + }); + + afterEach(() => { + appSlice.useCgroupRoot('/sys/fs/cgroup'); + fs.rmSync(root, { recursive: true, force: true }); + sinon.restore(); + }); + + describe('sliceName', () => { + it('names the app\'s slice under flux.slice', () => { + expect(appSlice.sliceName('palworld1790205120183')).to.equal('flux-palworld1790205120183.slice'); + }); + + it('escapes each hyphen, so one app\'s slice never nests inside another\'s', () => { + expect(appSlice.sliceName('my-app')).to.equal('flux-my\\x2dapp.slice'); + expect(appSlice.sliceName('my-app')).to.not.equal(`${appSlice.sliceName('my').slice(0, -'.slice'.length)}-app.slice`); + }); + + it('names no slice for a name no app can have', () => { + ['', 'a_b', 'a.b', '../x', 'a b', null, undefined].forEach((name) => { + expect(appSlice.sliceName(name), String(name)).to.equal(null); + }); + }); + }); + + describe('release', () => { + it('stops the app\'s slice once nothing lives in it', async () => { + slice('flux-my\\x2dapp.slice', false); + + await appSlice.release('my-app'); + + sinon.assert.calledOnceWithExactly(runCommand, 'systemctl', { runAsRoot: true, params: ['stop', 'flux-my\\x2dapp.slice'] }); + }); + + it('leaves a slice a process still lives in', async () => { + slice('flux-busy.slice', true); + + await appSlice.release('busy'); + + sinon.assert.notCalled(runCommand); + }); + + it('does nothing for an app with no slice', async () => { + await appSlice.release('absent'); + + sinon.assert.notCalled(runCommand); + }); + }); + + describe('stopEmptySlices', () => { + it('stops every empty app slice, and leaves those in use and anything else', async () => { + slice('flux-gone.slice', false); + slice('flux-gone\\x2dtoo.slice', false); + slice('flux-running.slice', true); + slice('other.slice', false); + + const count = await appSlice.stopEmptySlices(); + + expect(count).to.equal(2); + expect(stopped()).to.have.members(['flux-gone.slice', 'flux-gone\\x2dtoo.slice']); + }); + + it('counts no slice systemd could not stop', async () => { + slice('flux-gone.slice', false); + runCommand.resolves({ error: new Error('unit busy') }); + + expect(await appSlice.stopEmptySlices()).to.equal(0); + }); + + it('does nothing when there is no flux.slice', async () => { + expect(await appSlice.stopEmptySlices()).to.equal(0); + sinon.assert.notCalled(runCommand); + }); + }); +}); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 46fef3a905..05ea052076 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -17,8 +17,10 @@ describe('appStartupManager tests', () => { let globalStateStub; let appQueryServiceStub; let portManagerStub; + let appSliceStub; beforeEach(() => { + appSliceStub = { stopEmptySlices: sinon.stub().resolves(0) }; logStub = { info: sinon.stub(), warn: sinon.stub(), @@ -97,6 +99,7 @@ describe('appStartupManager tests', () => { './advancedWorkflows': advancedWorkflowsStub, '../appMonitoring/appReconciler': appReconcilerStub, './appUninstaller': appUninstallerStub, + './appSlice': appSliceStub, '../utils/globalState': globalStateStub, // runs the step again at once until it succeeds, as the real one does after a wait '../utils/untilSucceeds': { @@ -670,6 +673,25 @@ describe('appStartupManager tests', () => { }); describe('manageAppsOnBoot', () => { + it('stops every empty app slice before it removes any app', async () => { + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); + + await appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); + + sinon.assert.calledOnce(appSliceStub.stopEmptySlices); + sinon.assert.callOrder(appSliceStub.stopEmptySlices, appUninstallerStub.removeAppLocally); + }); + + it('stops every empty app slice before it reconciles the apps', async () => { + dockerServiceStub.dockerListContainers.resolves([]); + dbHelperStub.findInDatabase.resolves([]); + + await appStartupManager.manageAppsOnBoot({ machineRebooted: false, downtimeMs: 1000, cleanShutdown: true }); + + sinon.assert.calledOnce(appSliceStub.stopEmptySlices); + sinon.assert.callOrder(appSliceStub.stopEmptySlices, globalStateStub.waitForDbReady); + }); + it('should reconcile on FluxOS-only restart (no stopped containers)', async () => { const bootContext = { machineRebooted: false, downtimeMs: 1000, cleanShutdown: true, diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 54fc6e0a8a..44cffa8321 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -595,9 +595,11 @@ describe('appUninstaller tests', () => { // after successful teardown only (failed removals clear nothing), on forced // and unforced paths alike. let runtimeStateStub; + let appSliceStub; function buildUninstaller(spec, constantOverrides = {}, dbOverrides = {}) { runtimeStateStub = { remove: sinon.stub().resolves(), removeControllerState: sinon.stub().resolves() }; + appSliceStub = { release: sinon.stub().resolves() }; // The REAL departing tracker, taken fresh per test rather than // reimplemented here: a fake that counts differently from the module // would pass this suite over the defect the counting exists to prevent. @@ -713,6 +715,7 @@ describe('appUninstaller tests', () => { stopAppMonitoring: sinon.stub().resolves(), }, '../appManagement/appsRuntimeState': runtimeStateStub, + './appSlice': appSliceStub, 'node-cmd': { run: sinon.stub().callsFake((cmd, cb) => cb(null, '', '')) }, }); } @@ -927,6 +930,26 @@ describe('appUninstaller tests', () => { sinon.assert.calledWithExactly(onRemoved, 'comp2_testapp'); }); + it('releases the app\'s slice once the whole app is torn down', async () => { + const uninstaller = buildUninstaller(composedSpec); + + await uninstaller.removeAppLocally('testapp', res, true); + + sinon.assert.calledOnceWithExactly(appSliceStub.release, 'testapp'); + }); + + it('keeps the app\'s slice when one component is removed', async () => { + const uninstaller = buildUninstaller(composedSpec); + const onRemoved = sinon.stub(); + uninstaller.setOnComponentRemoved(onRemoved); + + await uninstaller.removeAppLocally('comp1_testapp', res, true); + + // the canary: the component was torn down + sinon.assert.calledOnceWithExactly(onRemoved, 'comp1_testapp'); + sinon.assert.notCalled(appSliceStub.release); + }); + it('scopes the seam to the one component on a component-scoped removal', async () => { const uninstaller = buildUninstaller(composedSpec); const onRemoved = sinon.stub(); diff --git a/tests/unit/dockerService.test.js b/tests/unit/dockerService.test.js index 5348a7fe94..f2788e048c 100644 --- a/tests/unit/dockerService.test.js +++ b/tests/unit/dockerService.test.js @@ -2330,6 +2330,30 @@ describe('dockerService tests', () => { await expect(dockerService.appDockerCreate(nodeApp, appName, true)).to.eventually.be.rejectedWith('Cannot read properties of undefined (reading \'forEach\')'); }); + + describe('the app\'s slice', () => { + const nodeApp = { + ...baseNodeApp, containerPorts: [], ports: [], version: 3, + }; + const created = () => dockerStub.firstCall.args[0]; + const driverIs = (CgroupDriver) => sinon.stub(Dockerode.prototype, 'info').resolves({ CgroupDriver, DriverStatus: [] }); + + it('puts the container in its app\'s slice under the systemd cgroup driver', async () => { + driverIs('systemd'); + + await dockerService.appDockerCreate(nodeApp, 'my-app', true); + + expect(created().HostConfig.CgroupParent).to.equal('flux-my\\x2dapp.slice'); + }); + + it('leaves the parent to docker under any other cgroup driver', async () => { + driverIs('cgroupfs'); + + await dockerService.appDockerCreate(nodeApp, 'my-app', true); + + expect(created().HostConfig).to.not.have.property('CgroupParent'); + }); + }); }); describe('loadImage tests', () => { const ID = 'sha256:1111111111111111111111111111111111111111111111111111111111111111'; From d8ea201531ac8d978633fd7c35b7e2b980ef38f7 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:37:09 +0100 Subject: [PATCH 202/369] fix(apps): a node out of service for a host stall removes every app, and none can run again The trip starts the removal at once. Each app gets its normal removal, which stops its containers before it tells the network, within five minutes; what of it can still run afterwards is killed through its cgroups and checked. An app whose every process is gone, or has SIGKILL pending, can never run its code again: a process the kernel holds keeps the kill and dies the moment the kernel lets it go. The apps are read from the database and from what is on disk, so an app the database cannot list is still found: its slice, and for a container created before slices, docker's own record of the container, read through sudo where FluxOS does not run as root. A regular DOS removal gives way to it, and so does boot; and every DOS removal moves past an app whose removal does not return within the deadline. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 3 + ZelBack/src/services/appLifecycle/appKill.js | 185 ++++++++++++++++++ .../appLifecycle/appStartupManager.js | 5 + .../appMonitoring/nodeStatusMonitor.js | 94 ++++++++- ZelBack/src/services/hostStallService.js | 22 +++ tests/unit/appKill.test.js | 136 +++++++++++++ tests/unit/appStartupManager.test.js | 14 ++ tests/unit/globalconfig/default.js | 3 + tests/unit/hostStallService.test.js | 39 ++++ tests/unit/nodeStatusMonitor.test.js | 132 +++++++++++++ 10 files changed, 631 insertions(+), 2 deletions(-) create mode 100644 ZelBack/src/services/appLifecycle/appKill.js create mode 100644 tests/unit/appKill.test.js diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index df9d4f6c10..c5f079d40c 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -46,6 +46,9 @@ module.exports = { hostStall: { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, + // How long one app's removal may take while the node is out of service + // before the next app's starts. + removalDeadlineMs: 5 * 60 * 1000, }, database: { url: '127.0.0.1', diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js new file mode 100644 index 0000000000..d41f8d8be5 --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -0,0 +1,185 @@ +/** + * Kill every process of an app through its cgroups, without docker or the + * database, and say whether any of them can still run. + * + * An app's containers live in its slice (appSlice). A container created before + * the slice existed lives in its runtime's scope, `system.slice/docker-.scope`, + * and the container's name is read from the runtime's own record on disk, which + * stays readable while docker does not answer. + * + * A process held in the kernel is not removed by a kill: it keeps the kill + * pending and dies the moment the kernel lets it go, without running the app's + * code again. So a cgroup whose every process has SIGKILL pending is as safe as + * an empty one. + */ + +const fs = require('fs').promises; +const path = require('path'); +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); +const appSlice = require('./appSlice'); +const { isArcane } = require('../utils/appConstants'); + +const SIGKILL_MASK = 1n << 8n; + +const DEFAULT_PATHS = { + cgroupRoot: '/sys/fs/cgroup', + procRoot: '/proc', + dockerRoot: isArcane ? '/dat/var/lib/docker' : '/var/lib/docker', +}; +let paths = { ...DEFAULT_PATHS }; + +/** + * @param {{cgroupRoot?: string, procRoot?: string, dockerRoot?: string}} options + * Where the cgroup filesystem, /proc and docker's data are; each one given replaces it. + */ +function usePaths(options) { + paths = { ...paths, ...options }; + appSlice.useCgroupRoot(paths.cgroupRoot); +} + +/** + * A file only root may read, read through sudo where FluxOS does not run as root. + * @param {string} file + * @returns {Promise} + */ +async function readAsRoot(file) { + const { stdout, error } = await serviceHelper.runCommand('cat', { runAsRoot: true, params: [file] }); + return error ? null : stdout; +} + +/** + * The app a container belongs to: `flux` for a single container, + * `flux_` for a component. + * @param {string} name The container's name. + * @returns {string|null} Null for a container that is not an app's. + */ +function appOf(name) { + if (!name.startsWith('flux')) return null; + return name.includes('_') ? name.slice(name.indexOf('_') + 1) : name.slice('flux'.length); +} + +/** + * @param {string} name The container's name. + * @param {string} appName + */ +function belongsTo(name, appName) { + return appOf(name) === appName; +} + +/** + * Every container docker has a record of, read from its records on disk. + * @returns {Promise>} Names without the leading '/'. + */ +async function containersOnDisk() { + const listing = await serviceHelper.runCommand('ls', { runAsRoot: true, params: ['-1', path.join(paths.dockerRoot, 'containers')] }); + if (listing.error) return []; + const ids = listing.stdout.split('\n').map((line) => line.trim()).filter((id) => /^[0-9a-f]{64}$/.test(id)); + const containers = []; + // eslint-disable-next-line no-restricted-syntax + for (const id of ids) { + // eslint-disable-next-line no-await-in-loop + const text = await readAsRoot(path.join(paths.dockerRoot, 'containers', id, 'config.v2.json')); + let name = null; + try { + ({ Name: name } = JSON.parse(text ?? 'null') ?? {}); + } catch { + name = null; + } + if (typeof name === 'string') containers.push({ id, name: name.replace(/^\//, '') }); + } + return containers; +} + +/** + * The scopes of the app's containers that live outside its slice. + * @param {string} appName + * @returns {Promise} Cgroup directories. + */ +async function legacyScopes(appName) { + return (await containersOnDisk()) + .filter(({ name }) => belongsTo(name, appName)) + .map(({ id }) => path.join(paths.cgroupRoot, 'system.slice', `docker-${id}.scope`)); +} + +/** + * Every cgroup that holds a container of the app and exists now. + * @param {string} appName + * @returns {Promise} + */ +async function cgroupsOf(appName) { + const unit = appSlice.sliceName(appName); + const candidates = [...(unit ? [appSlice.cgroupOf(unit)] : []), ...await legacyScopes(appName)]; + const present = await Promise.all(candidates.map((dir) => fs.access(path.join(dir, 'cgroup.procs')).then(() => dir, () => null))); + return present.filter(Boolean); +} + +/** + * Send SIGKILL to every process of the app. + * @param {string} appName + * @returns {Promise} The cgroups it was sent to. + */ +async function kill(appName) { + const cgroups = await cgroupsOf(appName); + // eslint-disable-next-line no-restricted-syntax + for (const dir of cgroups) { + // eslint-disable-next-line no-await-in-loop + const { error } = await serviceHelper.runCommand('sh', { runAsRoot: true, params: ['-c', 'echo 1 > "$1"', 'sh', path.join(dir, 'cgroup.kill')] }); + if (error) log.error(`appKill - ${appName}: the kill could not be sent to ${dir}: ${error.message}`); + } + return cgroups; +} + +/** + * @param {string} pid + * @returns {Promise} Whether the process has SIGKILL pending, or is gone. + */ +async function killPendingOrGone(pid) { + const status = await fs.readFile(path.join(paths.procRoot, pid, 'status'), 'utf8').catch(() => null); + if (status === null) return true; + return ['SigPnd', 'ShdPnd'].some((field) => { + const match = status.match(new RegExp(`^${field}:\\s*([0-9a-f]+)$`, 'm')); + return match ? (BigInt(`0x${match[1]}`) & SIGKILL_MASK) !== 0n : false; + }); +} + +/** + * Which of the app's processes could still run its code: every one that is + * neither gone nor has SIGKILL pending. + * @param {string} appName + * @returns {Promise} Their pids; empty when the app can never run again. + */ +async function stillRunnable(appName) { + const cgroups = await cgroupsOf(appName); + const pids = (await Promise.all(cgroups.map((dir) => fs.readFile(path.join(dir, 'cgroup.procs'), 'utf8').catch(() => '')))) + .flatMap((text) => text.split('\n').map((line) => line.trim()).filter(Boolean)); + const settled = await Promise.all(pids.map(killPendingOrGone)); + return pids.filter((pid, i) => !settled[i]); +} + +/** + * Every app with a slice or a container on this node, found without docker or + * the database. + * @returns {Promise} + */ +async function appsOnDisk() { + const names = new Set(); + const slices = await fs.readdir(path.join(paths.cgroupRoot, 'flux.slice')).catch(() => []); + slices.filter((entry) => /^flux-.+\.slice$/.test(entry)) + .forEach((entry) => names.add(entry.slice('flux-'.length, -'.slice'.length).replace(/\\x2d/g, '-'))); + (await containersOnDisk()).map(({ name }) => appOf(name)).filter(Boolean).forEach((name) => names.add(name)); + return [...names]; +} + +/** Restore the default paths. */ +function reset() { + usePaths(DEFAULT_PATHS); +} + +module.exports = { + kill, + stillRunnable, + appsOnDisk, + usePaths, + reset, +}; diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 225217ad7a..53516bf22a 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -429,6 +429,11 @@ async function manageAppsOnBoot(bootContext) { return; } + if (fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { + log.error('appStartupManager - Node is out of service for a host stall; its own removal unloads the apps'); + return; + } + if (fluxNetworkHelper.isNodeDos()) { log.error('appStartupManager - Node is in DOS state, removing all apps'); await removeAllApps('Node DOS'); diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index a08fe4b1c5..5df0043eab 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -8,10 +8,35 @@ const fluxCommunicationUtils = require('../fluxCommunicationUtils'); const messageStore = require('../appMessaging/messageStore'); const nodeConfirmationService = require('../nodeConfirmationService'); const networkStateService = require('../networkStateService'); +const hostStallService = require('../hostStallService'); +const appKill = require('../appLifecycle/appKill'); +const { RemovalOutcome } = require('../utils/removalOutcome'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); let removalInProgress = false; +let unloadingForHostStall = false; +/** What initialize was given, for the removal a host stall starts. */ +let deps = null; + +const REMOVAL_DEADLINE_MS = config.hostStall.removalDeadlineMs; +const TIMED_OUT = 'timedOut'; + +/** + * What `work` resolved, or TIMED_OUT once `ms` pass. The work itself keeps + * running: nothing here can stop a removal part way. + * @param {Promise<*>} work + * @param {number} ms + */ +async function withinDeadline(work, ms) { + let timer; + const deadline = new Promise((resolve) => { timer = setTimeout(resolve, ms, TIMED_OUT); }); + try { + return await Promise.race([work, deadline]); + } finally { + clearTimeout(timer); + } +} // Database collections const globalAppsLocations = config.database.appsglobal.collections.appsLocations; @@ -34,11 +59,18 @@ async function removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason) const appsInstalled = installedAppsRes.data; const canBroadcast = nodeConfirmationService.canSendMessages(); for (const installedApp of appsInstalled) { + // a host stall's own removal has the node, and this one gives way to it + if (unloadingForHostStall) break; log.info(`monitorNodeStatus - Application ${installedApp.name} going to be removed: ${reason}`); log.warn(`monitorNodeStatus - Removing application ${installedApp.name} locally`); + // One removal that never returns must not hold every other app here. // eslint-disable-next-line no-await-in-loop - await removeAppLocallyFn(installedApp.name, null, true, false, canBroadcast); - log.warn(`monitorNodeStatus - Application ${installedApp.name} locally removed`); + const outcome = await withinDeadline(removeAppLocallyFn(installedApp.name, null, true, false, canBroadcast), REMOVAL_DEADLINE_MS); + if (outcome === TIMED_OUT) { + log.error(`monitorNodeStatus - Application ${installedApp.name} was not removed within ${REMOVAL_DEADLINE_MS / 1000}s, moving on`); + } else { + log.warn(`monitorNodeStatus - Application ${installedApp.name} locally removed`); + } // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.nodeMonitorRemovalDelayMs ?? 60000); } @@ -63,7 +95,64 @@ async function removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, reason } } +/** + * Remove every app from a node whose host is stalled, and make sure none can + * run again. Each app gets its normal removal, which stops its containers before + * it tells the network, within REMOVAL_DEADLINE_MS. Whatever of it can still run + * afterwards is killed through its cgroups, and checked: an app whose every + * process is gone or has the kill pending can never run its code again. + * + * The apps are read from the database and from what is on disk, so an app the + * database cannot list is still found. Apps are not spaced out: every minute a + * stalled node keeps one is a minute its standby waits. + * @param {Function} installedAppsFn + * @param {Function} removeAppLocallyFn + */ +async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { + if (unloadingForHostStall) return; + unloadingForHostStall = true; + try { + const listed = await withinDeadline(installedAppsFn(), REMOVAL_DEADLINE_MS).catch(() => null); + const installed = listed?.status === 'success' ? listed.data.map((app) => app.name) : []; + const names = [...new Set([...installed, ...await appKill.appsOnDisk()])]; + const canBroadcast = nodeConfirmationService.canSendMessages(); + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + log.warn(`monitorNodeStatus - Application ${name} going to be removed: host stall`); + // eslint-disable-next-line no-await-in-loop + const outcome = await withinDeadline(removeAppLocallyFn(name, null, true, false, canBroadcast), REMOVAL_DEADLINE_MS) + .catch(() => RemovalOutcome.FAILED); + // eslint-disable-next-line no-await-in-loop + if ((await appKill.stillRunnable(name)).length) { + log.error(`monitorNodeStatus - Application ${name} still runs after its removal (${outcome}), killing it`); + // eslint-disable-next-line no-await-in-loop + await appKill.kill(name); + // eslint-disable-next-line no-await-in-loop + const left = await appKill.stillRunnable(name); + if (left.length) { + log.error(`monitorNodeStatus - Application ${name} could not be stopped: processes ${left.join(', ')} can still run`); + } else { + log.warn(`monitorNodeStatus - Application ${name} can no longer run`); + } + } + } + } finally { + unloadingForHostStall = false; + } +} + +/** The removal a host stall starts. Nothing awaits it, so its failure is logged here. */ +async function unloadOnHostStall() { + try { + await unloadForHostStall(deps.installedAppsFn, deps.removeAppLocallyFn); + } catch (error) { + log.error(`monitorNodeStatus - app removal (host stall) failed: ${error.message}`); + } +} + function initialize(installedAppsFn, removeAppLocallyFn) { + deps = { installedAppsFn, removeAppLocallyFn }; + hostStallService.onTrip(unloadOnHostStall); nodeConfirmationService.onConfirmationChange((confirmed) => { if (!confirmed) { log.info('monitorNodeStatus - Confirmation lost, triggering immediate app removal'); @@ -176,4 +265,5 @@ async function monitorNodeStatus(installedAppsFn, removeAppLocallyFn) { module.exports = { initialize, monitorNodeStatus, + unloadForHostStall, }; diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index f26dd24ddf..35ae5ac953 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -39,6 +39,8 @@ const STUCK_INTERVALS = config.hostStall.stuckIntervals; */ let tracked = new Map(); let readTimer = null; +/** Called once the node is out of service for a host stall. */ +const tripListeners = []; const DEFAULT_PATHS = { procRoot: '/proc', recordFile: path.join(fluxDirPath, 'config', 'hostStall.json'), @@ -166,6 +168,22 @@ async function recordTrip(reason) { } } +/** Tell every listener the node is out of service for a host stall. */ +function announceTrip() { + tripListeners.forEach((listener) => listener()); +} + +/** + * Call `listener` once the node is out of service for a host stall: at once + * when it already is, whether it tripped in this process or the trip was + * restored at start. + * @param {function(): void} listener + */ +function onTrip(listener) { + tripListeners.push(listener); + if (fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) listener(); +} + /** Restore a trip recorded in this boot, and discard one from an earlier boot. */ async function restoreTrip() { try { @@ -180,6 +198,7 @@ async function restoreTrip() { return; } fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); + announceTrip(); } catch (error) { log.error(`hostStall - the recorded trip could not be read: ${error.message}`); } @@ -197,6 +216,7 @@ async function act() { const reason = `Host stall: ${describe(longest)}`; fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); await recordTrip(reason); + announceTrip(); } } @@ -238,10 +258,12 @@ function stop() { readTimer = null; tracked = new Map(); paths = { ...DEFAULT_PATHS }; + tripListeners.length = 0; } module.exports = { start, stop, + onTrip, readOnce, }; diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js new file mode 100644 index 0000000000..a964c33980 --- /dev/null +++ b/tests/unit/appKill.test.js @@ -0,0 +1,136 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const appKill = require('../../ZelBack/src/services/appLifecycle/appKill'); + +describe('appKill tests', () => { + let root; + let cgroupRoot; + let procRoot; + let dockerRoot; + let killed; + + const id = (n) => String(n).repeat(64).slice(0, 64); + + function cgroup(rel, pids) { + const dir = path.join(cgroupRoot, rel); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'cgroup.procs'), pids.map((pid) => `${pid}\n`).join('')); + fs.writeFileSync(path.join(dir, 'cgroup.kill'), ''); + return dir; + } + function container(n, name, pids) { + fs.mkdirSync(path.join(dockerRoot, 'containers', id(n)), { recursive: true }); + fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pids[0] ?? 0) } })); + return cgroup(path.join('system.slice', `docker-${id(n)}.scope`), pids); + } + function processWith(pid, { sigPnd = '0000000000000000', shdPnd = '0000000000000000' } = {}) { + fs.mkdirSync(path.join(procRoot, pid), { recursive: true }); + fs.writeFileSync(path.join(procRoot, pid, 'status'), `Name:\tapp\nSigPnd:\t${sigPnd}\nShdPnd:\t${shdPnd}\n`); + } + + // what sudo would do: read and list as root, and write to cgroup.kill + function asRoot(cmd, { params }) { + if (cmd === 'ls') return Promise.resolve({ stdout: fs.readdirSync(params[1]).join('\n'), error: null }); + if (cmd === 'cat') { + try { + return Promise.resolve({ stdout: fs.readFileSync(params[0], 'utf8'), error: null }); + } catch (error) { + return Promise.resolve({ stdout: '', error }); + } + } + if (cmd === 'sh') { + killed.push(path.dirname(params[3])); + return Promise.resolve({ stdout: '', error: null }); + } + return Promise.resolve({ stdout: '', error: new Error(`unexpected ${cmd}`) }); + } + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'appkill-')); + cgroupRoot = path.join(root, 'cgroup'); + procRoot = path.join(root, 'proc'); + dockerRoot = path.join(root, 'docker'); + fs.mkdirSync(path.join(dockerRoot, 'containers'), { recursive: true }); + appKill.usePaths({ cgroupRoot, procRoot, dockerRoot }); + killed = []; + sinon.stub(serviceHelper, 'runCommand').callsFake(asRoot); + }); + + afterEach(() => { + appKill.reset(); + fs.rmSync(root, { recursive: true, force: true }); + sinon.restore(); + }); + + describe('kill', () => { + it('kills the app\'s slice and the scopes of its containers made before slices, and no other app\'s', async () => { + const slice = cgroup(path.join('flux.slice', 'flux-my\\x2dapp.slice'), ['10']); + const web = container(1, 'fluxweb_my-app', ['11']); + const single = container(2, 'fluxmy-app', ['12']); + container(3, 'fluxweb_my-app2', ['13']); + container(4, 'fluxweb_other', ['14']); + cgroup(path.join('flux.slice', 'flux-my.slice'), ['15']); + + const sent = await appKill.kill('my-app'); + + expect(sent).to.have.members([slice, web, single]); + expect(killed).to.have.members([slice, web, single]); + }); + + it('sends nothing for an app with no cgroup left', async () => { + fs.mkdirSync(path.join(dockerRoot, 'containers', id(5)), { recursive: true }); + fs.writeFileSync(path.join(dockerRoot, 'containers', id(5), 'config.v2.json'), JSON.stringify({ Name: '/fluxgone' })); + + expect(await appKill.kill('gone')).to.deep.equal([]); + expect(killed).to.deep.equal([]); + }); + + it('passes over a container record it cannot read', async () => { + const web = container(1, 'fluxweb_app', ['11']); + fs.mkdirSync(path.join(dockerRoot, 'containers', id(6)), { recursive: true }); + fs.writeFileSync(path.join(dockerRoot, 'containers', id(6), 'config.v2.json'), '{ broken'); + + expect(await appKill.kill('app')).to.deep.equal([web]); + }); + }); + + describe('stillRunnable', () => { + beforeEach(() => { + cgroup(path.join('flux.slice', 'flux-app.slice'), ['21', '22', '23', '24']); + }); + + it('names each process that is neither gone nor has the kill pending', async () => { + processWith('21'); + processWith('22', { shdPnd: '0000000000000100' }); + processWith('23', { sigPnd: '0000000000004100' }); + // 24 is gone + + expect(await appKill.stillRunnable('app')).to.deep.equal(['21']); + }); + + it('does not take another pending signal for the kill', async () => { + ['21', '22', '23', '24'].forEach((pid) => processWith(pid, { shdPnd: '0000000000004000' })); + + expect(await appKill.stillRunnable('app')).to.deep.equal(['21', '22', '23', '24']); + }); + + it('names nothing for an app with no cgroup left', async () => { + expect(await appKill.stillRunnable('other')).to.deep.equal([]); + }); + }); + + describe('appsOnDisk', () => { + it('names every app with a slice or a container record, and no container that is not an app\'s', async () => { + cgroup(path.join('flux.slice', 'flux-sliced\\x2dapp.slice'), []); + container(1, 'fluxweb_composed', ['11']); + container(2, 'fluxsingle', ['12']); + container(3, 'watchtower', ['13']); + + expect(await appKill.appsOnDisk()).to.have.members(['sliced-app', 'composed', 'single']); + }); + }); +}); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 05ea052076..002ead956b 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -39,6 +39,8 @@ describe('appStartupManager tests', () => { fluxNetworkHelperStub = { getLocalSocketAddress: sinon.stub(), isNodeDos: sinon.stub().returns(false), + isStickyDosHeldBy: sinon.stub().returns(false), + StickyDosOwner: { HOST_STALL: 'hostStall' }, allowAppPorts: sinon.stub().resolves({ failed: [], locked: false }), }; @@ -673,6 +675,18 @@ describe('appStartupManager tests', () => { }); describe('manageAppsOnBoot', () => { + it('leaves a node out of service for a host stall to its own removal, and starts no app', async () => { + fluxNetworkHelperStub.isStickyDosHeldBy.returns(true); + fluxNetworkHelperStub.isNodeDos.returns(true); + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); + + await appStartupManager.manageAppsOnBoot({ machineRebooted: false, downtimeMs: 1000, cleanShutdown: true }); + + sinon.assert.calledWithExactly(fluxNetworkHelperStub.isStickyDosHeldBy, 'hostStall'); + sinon.assert.notCalled(appUninstallerStub.removeAppLocally); + sinon.assert.notCalled(globalStateStub.waitForDbReady); + }); + it('stops every empty app slice before it removes any app', async () => { appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 9085bccb3f..41c5501353 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -32,6 +32,9 @@ module.exports = { hostStall: { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, + // How long one app's removal may take while the node is out of service + // before the next app's starts. + removalDeadlineMs: 5 * 60 * 1000, }, database: { url: database, diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index aeb1848b90..ad9c698afd 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -171,6 +171,45 @@ describe('hostStallService tests', () => { expect(outOfService()).to.equal(true); }); + describe('onTrip', () => { + it('tells a listener the moment the node goes out of service, and not before', async () => { + const listener = sinon.stub(); + hostStallService.onTrip(listener); + stuckMongod(); + await readTimes(stuckIntervals); + sinon.assert.notCalled(listener); + + await read(); + + sinon.assert.calledOnce(listener); + await readTimes(2); + sinon.assert.calledOnce(listener); + }); + + it('tells a listener at once when the node is already out of service', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + const listener = sinon.stub(); + + hostStallService.onTrip(listener); + + sinon.assert.calledOnce(listener); + }); + + it('tells a listener when a trip recorded in this boot is restored at start', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + const listener = sinon.stub(); + hostStallService.onTrip(listener); + + await hostStallService.start(paths); + + sinon.assert.calledOnce(listener); + }); + }); + describe('the trip record', () => { async function trip() { stuckMongod(); diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 9166d325f8..611deb1c1e 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -24,6 +24,7 @@ describe('nodeStatusMonitor tests', () => { }, }, config: { + hostStall: { removalDeadlineMs: 300000 }, database: { appsglobal: { database: 'globalapps', @@ -114,4 +115,135 @@ describe('nodeStatusMonitor tests', () => { sinon.assert.calledWithMatch(logStub.error, /daemon unreachable\) failed/); }); }); + + describe('removal while out of service', () => { + const DEADLINE = 300000; + let clock; + let monitor; + let appKill; + let onTrip; + let dos; + const never = () => new Promise(() => {}); + const installed = (...names) => sinon.stub().resolves({ status: 'success', data: names.map((name) => ({ name })) }); + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + dos = false; + appKill = { + appsOnDisk: sinon.stub().resolves([]), + stillRunnable: sinon.stub().resolves([]), + kill: sinon.stub().resolves([]), + }; + monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { + config: { hostStall: { removalDeadlineMs: DEADLINE }, fluxapps: { nodeMonitorRemovalDelayMs: 0 }, database: { appsglobal: { collections: {} } } }, + '../dbHelper': {}, + // the spacing between apps passes, and the wait before the next pass never ends + '../serviceHelper': { delay: sinon.stub().callsFake((ms) => (ms === 0 ? Promise.resolve() : never())) }, + '../fluxNetworkHelper': { isNodeDos: () => dos }, + '../fluxCommunicationUtils': {}, + '../appMessaging/messageStore': {}, + '../networkStateService': {}, + '../nodeConfirmationService': { + onConfirmationChange: sinon.stub(), onDaemonStale: sinon.stub(), canSendMessages: () => true, + }, + '../hostStallService': { onTrip: (listener) => { onTrip = listener; } }, + '../appLifecycle/appKill': appKill, + '../../lib/log': logStub, + }); + }); + + afterEach(() => { + clock.restore(); + }); + + it('removes every app once the host stalls, starting from the trip', async () => { + const remove = sinon.stub().resolves('removed'); + monitor.initialize(installed('one', 'two'), remove); + + onTrip(); + await clock.tickAsync(0); + + expect(remove.getCalls().map((call) => call.args[0])).to.deep.equal(['one', 'two']); + sinon.assert.notCalled(appKill.kill); + }); + + it('kills an app whose removal does not return within the deadline, and moves on', async () => { + const remove = sinon.stub(); + remove.withArgs('stuck').callsFake(never); + remove.withArgs('next').resolves('removed'); + appKill.stillRunnable.withArgs('stuck').onFirstCall().resolves(['41']).onSecondCall().resolves([]); + + const unload = monitor.unloadForHostStall(installed('stuck', 'next'), remove); + await clock.tickAsync(DEADLINE - 1); + sinon.assert.notCalled(appKill.kill); + await clock.tickAsync(1); + await unload; + + sinon.assert.calledOnceWithExactly(appKill.kill, 'stuck'); + sinon.assert.calledWith(remove, 'next'); + sinon.assert.calledWithMatch(logStub.warn, /stuck can no longer run/); + }); + + it('kills an app whose removal failed while its processes can still run', async () => { + const remove = sinon.stub().resolves('failed'); + appKill.stillRunnable.onFirstCall().resolves(['41']).onSecondCall().resolves([]); + + await monitor.unloadForHostStall(installed('app'), remove); + + sinon.assert.calledOnceWithExactly(appKill.kill, 'app'); + }); + + it('names the processes a kill could not stop', async () => { + appKill.stillRunnable.resolves(['41', '42']); + + await monitor.unloadForHostStall(installed('app'), sinon.stub().resolves('failed')); + + sinon.assert.calledWithMatch(logStub.error, /app could not be stopped: processes 41, 42 can still run/); + }); + + it('removes the apps found on disk when the database cannot list them', async () => { + appKill.appsOnDisk.resolves(['ondisk']); + const remove = sinon.stub().resolves('removed'); + + await monitor.unloadForHostStall(sinon.stub().rejects(new Error('timed out')), remove); + + sinon.assert.calledOnceWithMatch(remove, 'ondisk'); + }); + + it('removes each app once when the database and the disk both name it', async () => { + appKill.appsOnDisk.resolves(['both']); + const remove = sinon.stub().resolves('removed'); + + await monitor.unloadForHostStall(installed('both'), remove); + + sinon.assert.calledOnce(remove); + }); + + it('moves past a DOS removal that does not return within the deadline', async () => { + dos = true; + const remove = sinon.stub(); + remove.withArgs('stuck').callsFake(never); + remove.withArgs('next').resolves('removed'); + + monitor.monitorNodeStatus(installed('stuck', 'next'), remove); + await clock.tickAsync(DEADLINE); + + sinon.assert.calledWith(remove, 'next'); + sinon.assert.calledWithMatch(logStub.error, /stuck was not removed within 300s, moving on/); + }); + + it('starts no DOS removal while a host stall\'s removal has the node', async () => { + dos = true; + const remove = sinon.stub().callsFake(never); + const unload = monitor.unloadForHostStall(installed('held'), remove); + await clock.tickAsync(0); + + monitor.monitorNodeStatus(installed('other'), remove); + await clock.tickAsync(0); + + sinon.assert.neverCalledWith(remove, 'other'); + await clock.tickAsync(DEADLINE); + await unload; + }); + }); }); From c8b339e40febdf7c2cb41162fb6f4c1034b43822 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:40:55 +0100 Subject: [PATCH 203/369] test(db): the database-suspect driver case runs well inside mocha's default timeout Co-Authored-By: Claude Opus 5.5 --- tests/unit/databaseHealthService.test.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/unit/databaseHealthService.test.js b/tests/unit/databaseHealthService.test.js index 08ef203bbe..7910c542b8 100644 --- a/tests/unit/databaseHealthService.test.js +++ b/tests/unit/databaseHealthService.test.js @@ -131,7 +131,7 @@ describe('databaseHealthService tests', () => { beforeEach(async () => { clock.restore(); databaseHealthService.stop(); - real = new MongoClient(`mongodb://${config.database.url}:${config.database.port}`, { socketTimeoutMS: 300 }); + real = new MongoClient(`mongodb://${config.database.url}:${config.database.port}`, { socketTimeoutMS: 100 }); await real.connect(); databaseHealthService.watch(real, 'probe'); }); @@ -140,11 +140,12 @@ describe('databaseHealthService tests', () => { await real.close(); }); - it('counts one stuck read as two round trips, so two stuck reads hold the node', async () => { + it('counts one stuck read as two round trips, so two stuck reads hold the node', async function stuckReads() { + this.timeout(10000); const ranOut = sinon.spy(log, 'warn'); const collection = real.db(config.database.local.database).collection('databasehealthprobe'); await collection.insertOne({ probe: 1 }); - const stuckRead = () => collection.findOne({ $where: 'sleep(1000) || true' }).catch((error) => error); + const stuckRead = () => collection.findOne({ $where: 'sleep(400) || true' }).catch((error) => error); const first = await stuckRead(); // the canary: the read did run out, and the driver retried it once @@ -155,7 +156,8 @@ describe('databaseHealthService tests', () => { await stuckRead(); expect(fluxNetworkHelper.isPlacementHeld()).to.equal(true); - await new Promise((resolve) => { setTimeout(resolve, 1200); }); + // the server's own sleeps end before the collection is dropped + await new Promise((resolve) => { setTimeout(resolve, 450); }); await collection.drop(); }); }); From d4f4f537f04a3858df108f6539155bb50978c353 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:50:40 +0100 Subject: [PATCH 204/369] fix(apps): boot moves past an app whose removal does not return within five minutes When boot removes every app, each removal has fluxapps.appRemovalDeadlineMs; one that does not return is logged and the next app's starts, so one hung removal no longer holds boot and every app after it. The deadline is shared with the DOS removals through utils/withinDeadline. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 6 +-- .../appLifecycle/appStartupManager.js | 5 ++- .../appMonitoring/nodeStatusMonitor.js | 20 +--------- ZelBack/src/services/utils/withinDeadline.js | 24 ++++++++++++ tests/unit/appStartupManager.test.js | 20 ++++++++++ tests/unit/globalconfig/default.js | 6 +-- tests/unit/nodeStatusMonitor.test.js | 4 +- tests/unit/withinDeadline.test.js | 38 +++++++++++++++++++ 8 files changed, 96 insertions(+), 27 deletions(-) create mode 100644 ZelBack/src/services/utils/withinDeadline.js create mode 100644 tests/unit/withinDeadline.test.js diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index c5f079d40c..5edf694e9b 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -46,9 +46,6 @@ module.exports = { hostStall: { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, - // How long one app's removal may take while the node is out of service - // before the next app's starts. - removalDeadlineMs: 5 * 60 * 1000, }, database: { url: '127.0.0.1', @@ -608,6 +605,9 @@ module.exports = { bootDelayMultiplier: 1, spawnDelayMs: 0, removalSpacingMs: 60000, + // How long one app's removal may take, when every app is removed, before the + // next app's starts. A removal past it keeps running. + appRemovalDeadlineMs: 5 * 60 * 1000, // Per-document expiry for the ephemeral app collections, in seconds, read by // appConstants.js. Each record carries its own deadline and the collection // index is expireAt/expireAfterSeconds:0, so changing one of these takes diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 53516bf22a..8a877ba949 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -17,6 +17,7 @@ const appReconciler = require('../appMonitoring/appReconciler'); const appUninstaller = require('./appUninstaller'); const appNetworkLinker = require('./appNetworkLinker'); const appSlice = require('./appSlice'); +const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); const portManager = require('../appNetwork/portManager'); const globalState = require('../utils/globalState'); const fluxEventBus = require('../utils/fluxEventBus'); @@ -384,8 +385,10 @@ async function removeAllApps(reason) { if (installedAppsRes.status === 'success') { for (const app of installedAppsRes.data) { log.warn(`REMOVAL REASON: ${reason} - removing ${app.name}`); + // One removal that never returns must not hold boot, or every app after it. // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(app.name, null, true, false, true); + const outcome = await withinDeadline(appUninstaller.removeAppLocally(app.name, null, true, false, true), config.fluxapps.appRemovalDeadlineMs); + if (outcome === TIMED_OUT) log.error(`appStartupManager - ${app.name} was not removed within ${config.fluxapps.appRemovalDeadlineMs / 1000}s, moving on`); } } } diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 5df0043eab..422866541e 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -11,6 +11,7 @@ const networkStateService = require('../networkStateService'); const hostStallService = require('../hostStallService'); const appKill = require('../appLifecycle/appKill'); const { RemovalOutcome } = require('../utils/removalOutcome'); +const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -19,24 +20,7 @@ let unloadingForHostStall = false; /** What initialize was given, for the removal a host stall starts. */ let deps = null; -const REMOVAL_DEADLINE_MS = config.hostStall.removalDeadlineMs; -const TIMED_OUT = 'timedOut'; - -/** - * What `work` resolved, or TIMED_OUT once `ms` pass. The work itself keeps - * running: nothing here can stop a removal part way. - * @param {Promise<*>} work - * @param {number} ms - */ -async function withinDeadline(work, ms) { - let timer; - const deadline = new Promise((resolve) => { timer = setTimeout(resolve, ms, TIMED_OUT); }); - try { - return await Promise.race([work, deadline]); - } finally { - clearTimeout(timer); - } -} +const REMOVAL_DEADLINE_MS = config.fluxapps.appRemovalDeadlineMs; // Database collections const globalAppsLocations = config.database.appsglobal.collections.appsLocations; diff --git a/ZelBack/src/services/utils/withinDeadline.js b/ZelBack/src/services/utils/withinDeadline.js new file mode 100644 index 0000000000..d82185e422 --- /dev/null +++ b/ZelBack/src/services/utils/withinDeadline.js @@ -0,0 +1,24 @@ +/** What withinDeadline answers when the deadline passed first. */ +const TIMED_OUT = 'timedOut'; + +/** + * What `work` resolved, or TIMED_OUT once `ms` pass. The work itself keeps + * running: this only stops the caller waiting for it. + * @param {Promise<*>} work + * @param {number} ms + * @returns {Promise<*>} + */ +async function withinDeadline(work, ms) { + let timer; + const deadline = new Promise((resolve) => { timer = setTimeout(resolve, ms, TIMED_OUT); }); + try { + return await Promise.race([work, deadline]); + } finally { + clearTimeout(timer); + } +} + +module.exports = { + TIMED_OUT, + withinDeadline, +}; diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 002ead956b..95f6141c28 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai'); const sinon = require('sinon'); +const config = require('config'); const { resetGlobalState } = require('./fixtures/globalState'); const proxyquire = require('proxyquire').noCallThru(); @@ -687,6 +688,25 @@ describe('appStartupManager tests', () => { sinon.assert.notCalled(globalStateStub.waitForDbReady); }); + it('moves past an app whose removal does not return within the deadline, and boot still settles', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'stuck' }, { name: 'next' }] }); + appUninstallerStub.removeAppLocally.withArgs('stuck').returns(new Promise(() => {})); + + const boot = appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); + await clock.tickAsync(config.fluxapps.appRemovalDeadlineMs - 1); + sinon.assert.neverCalledWith(appUninstallerStub.removeAppLocally, 'next'); + await clock.tickAsync(1); + await boot; + + sinon.assert.calledWith(appUninstallerStub.removeAppLocally, 'next'); + expect(globalStateStub.bootContainerStateSettled).to.equal(true); + } finally { + clock.restore(); + } + }); + it('stops every empty app slice before it removes any app', async () => { appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 41c5501353..5c4e6b90cd 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -32,9 +32,6 @@ module.exports = { hostStall: { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, - // How long one app's removal may take while the node is out of service - // before the next app's starts. - removalDeadlineMs: 5 * 60 * 1000, }, database: { url: database, @@ -318,6 +315,9 @@ module.exports = { bootDelayMultiplier: 1, spawnDelayMs: 0, removalSpacingMs: 60000, + // How long one app's removal may take, when every app is removed, before the + // next app's starts. A removal past it keeps running. + appRemovalDeadlineMs: 5 * 60 * 1000, locationTtlS: 7500, installingTtlS: 900, installErrorTtlS: 86400, diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 611deb1c1e..f222aa4ae7 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -24,7 +24,7 @@ describe('nodeStatusMonitor tests', () => { }, }, config: { - hostStall: { removalDeadlineMs: 300000 }, + fluxapps: { appRemovalDeadlineMs: 300000 }, database: { appsglobal: { database: 'globalapps', @@ -135,7 +135,7 @@ describe('nodeStatusMonitor tests', () => { kill: sinon.stub().resolves([]), }; monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { - config: { hostStall: { removalDeadlineMs: DEADLINE }, fluxapps: { nodeMonitorRemovalDelayMs: 0 }, database: { appsglobal: { collections: {} } } }, + config: { fluxapps: { appRemovalDeadlineMs: DEADLINE, nodeMonitorRemovalDelayMs: 0 }, database: { appsglobal: { collections: {} } } }, '../dbHelper': {}, // the spacing between apps passes, and the wait before the next pass never ends '../serviceHelper': { delay: sinon.stub().callsFake((ms) => (ms === 0 ? Promise.resolve() : never())) }, diff --git a/tests/unit/withinDeadline.test.js b/tests/unit/withinDeadline.test.js new file mode 100644 index 0000000000..457a7ce411 --- /dev/null +++ b/tests/unit/withinDeadline.test.js @@ -0,0 +1,38 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const { withinDeadline, TIMED_OUT } = require('../../ZelBack/src/services/utils/withinDeadline'); + +describe('withinDeadline tests', () => { + let clock; + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + }); + + afterEach(() => { + clock.restore(); + }); + + it('answers what the work resolved, when it resolves in time', async () => { + expect(await withinDeadline(Promise.resolve('done'), 1000)).to.equal('done'); + }); + + it('rejects with the work\'s own error, when it rejects in time', async () => { + const failure = await withinDeadline(Promise.reject(new Error('refused')), 1000).catch((error) => error); + + expect(failure.message).to.equal('refused'); + }); + + it('answers TIMED_OUT once the deadline passes first', async () => { + const answer = withinDeadline(new Promise(() => {}), 1000); + await clock.tickAsync(1000); + + expect(await answer).to.equal(TIMED_OUT); + }); + + it('leaves no timer behind once the work resolves', async () => { + await withinDeadline(Promise.resolve('done'), 1000); + + expect(clock.countTimers()).to.equal(0); + }); +}); From 336c1dfd13209eaf365e0de05ae966af9bd7261a Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 11:55:18 +0100 Subject: [PATCH 205/369] fix(apps): a kill finds docker's records where docker says they are Boot reads docker's data root from docker info once docker answers, so a container created before slices is found on a node whose docker keeps its data elsewhere than its kind's default. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 3 ++- ZelBack/src/services/serviceManager.js | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index d41f8d8be5..aa0b2a0f78 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -5,7 +5,8 @@ * An app's containers live in its slice (appSlice). A container created before * the slice existed lives in its runtime's scope, `system.slice/docker-.scope`, * and the container's name is read from the runtime's own record on disk, which - * stays readable while docker does not answer. + * stays readable while docker does not answer. Where those records are is read + * from docker at boot; until then it is docker's default for the node's kind. * * A process held in the kernel is not removed by a kill: it keeps the kill * pending and dies the moment the kernel lets it go, without running the app's diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 103566537f..4746a9b93d 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -13,6 +13,7 @@ const networkStateService = require('./networkStateService'); const fluxNetworkHelper = require('./fluxNetworkHelper'); const databaseHealthService = require('./databaseHealthService'); const hostStallService = require('./hostStallService'); +const appKill = require('./appLifecycle/appKill'); // App modular services - replacing appsService const appInstaller = require('./appLifecycle/appInstaller'); const appUninstaller = require('./appLifecycle/appUninstaller'); @@ -276,6 +277,10 @@ async function startFluxFunctions() { // DOS, so /flux/info names the version found and the version required. fluxNetworkHelper.checkMongoDbVersionAllowed(await dbHelper.getMongoDbVersion()); await dockerService.waitForDocker(); + // Where docker keeps its records, read while it answers, for a kill made + // while it does not. + const dockerRoot = await dockerService.dockerInfo().then((info) => info?.DockerRootDir, () => null); + if (dockerRoot) appKill.usePaths({ dockerRoot }); // As soon as Docker answers, before the container firewall is applied: a // below-floor node stays up holding a sticky DOS, so /flux/info names the // version found and the version required. From e99c65dde929374a841221a343bd42a930700408 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:01:47 +0100 Subject: [PATCH 206/369] test(harness): a node's docker data can be frozen, holding docker in the kernel createTestEnv({ freezableDockerRoot: [index] }) mounts the node's docker data root on a filesystem of its own, an image on the node's volume mounted before docker starts. framework/host-stall.js freezes and thaws it, and reads which threads of a process sat in uninterruptible sleep without running across an interval. The env's teardown thaws every node before it stops them. A seeded test app can log on an interval, which docker writes to its data root. Co-Authored-By: Claude Opus 5.5 --- test-infra/entrypoint.sh | 11 ++++ test-infra/runner/framework/host-stall.js | 60 +++++++++++++++++++ .../runner/framework/reconciler-suite.js | 8 ++- test-infra/runner/framework/seed-helper.js | 4 +- test-infra/runner/framework/test-env.js | 16 +++-- 5 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 test-infra/runner/framework/host-stall.js diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index e330cfbf27..72b0927994 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -52,6 +52,17 @@ mkdir -p /dat/var/lib/fluxd \ /dat/usr/lib/fluxwatchdog \ /mnt/appdata/flux-apps +# A node whose docker data a suite freezes (framework/host-stall.js) keeps it on a +# filesystem of its own, mounted before docker starts. Freezing it holds every write +# docker makes there in the kernel, which is what a stalled host does to docker. The +# image lives on the node's volume, so a restarted node mounts the same data again. +if [ "$FLUX_FREEZABLE_DOCKER_ROOT" = "true" ]; then + img=/mnt/appdata/docker-root.img + [ -f "$img" ] || { truncate -s 8G "$img" && mkfs.ext4 -q -F "$img"; } + mkdir -p /mnt/appdata/docker + mountpoint -q /mnt/appdata/docker || mount -o loop "$img" /mnt/appdata/docker +fi + # In stub mode FluxOS only needs somewhere to read an API key from; the calls # themselves go to the shared stub. In binary mode the config is syncthing's own # and this fixture must not be in the way of it. diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js new file mode 100644 index 0000000000..390de7686a --- /dev/null +++ b/test-infra/runner/framework/host-stall.js @@ -0,0 +1,60 @@ +// A stalled host, on a real kernel: the filesystem a node's docker keeps its data on, +// frozen. Every write docker makes there is then held in the kernel - its threads sit +// in uninterruptible sleep and never run - until the filesystem is thawed, which is +// what a host that stops returning the disk does to docker. +// +// The node is booted with createTestEnv({ freezableDockerRoot: [index] }), which +// mounts its docker data root on a filesystem of its own. The env's teardown thaws +// it, so a node a failing test left frozen can still be stopped. +import { execInContainer } from './container.js'; + +export const DOCKER_ROOT = '/mnt/appdata/docker'; + +async function run(container, command, what) { + const result = await execInContainer(container, command); + if (result.exitCode !== 0) throw new Error(`${what}: ${(result.output || result.stderr || '').trim()}`); + return result.stdout; +} + +/** Hold every write the node's docker makes to its data root, until thawDocker. */ +export async function freezeDocker(container) { + await run(container, `mountpoint -q ${DOCKER_ROOT}`, `${DOCKER_ROOT} is not a mount of its own: boot the node with freezableDockerRoot`); + await run(container, `fsfreeze -f ${DOCKER_ROOT}`, 'freezeDocker'); +} + +/** Let the node's docker write again. */ +export async function thawDocker(container) { + await run(container, `fsfreeze -u ${DOCKER_ROOT}`, 'thawDocker'); +} + +/** + * One reading of every thread of the processes named `comm` in the node: its + * state, its voluntary plus nonvoluntary context switches, and where it waits. + * @returns {Promise>} By thread id. + */ +async function readThreads(container, comm) { + const script = `for p in $(pidof ${comm}); do for t in /proc/$p/task/*; do ` + + 'echo "${t##*/} $(cut -d" " -f3 $t/stat) $(awk \'/ctxt_switches/{s+=$2} END{print s}\' $t/status) $(cat $t/wchan)"; ' + + 'done; done; true'; + const out = await run(container, script, `reading ${comm}'s threads`); + return new Map(out.split('\n').filter(Boolean).map((line) => { + const [tid, state, switches, wchan] = line.split(' '); + return [tid, { state, switches: Number(switches), wchan }]; + })); +} + +/** + * The threads of `comm` that did not run at all across `overMs`: in D at both + * readings, with the same switch count. What FluxOS's detector reads as stuck, + * read independently of it. + * @returns {Promise>} + */ +export async function stuckThreads(container, comm, { overMs = 3000 } = {}) { + const first = await readThreads(container, comm); + await new Promise((resolve) => { setTimeout(resolve, overMs); }); + const second = await readThreads(container, comm); + return [...second].filter(([tid, now]) => { + const before = first.get(tid); + return before && before.state === 'D' && now.state === 'D' && before.switches === now.switches; + }).map(([tid, now]) => ({ tid, wchan: now.wchan })); +} diff --git a/test-infra/runner/framework/reconciler-suite.js b/test-infra/runner/framework/reconciler-suite.js index 75111dd97c..6e7ba2d8a5 100644 --- a/test-infra/runner/framework/reconciler-suite.js +++ b/test-infra/runner/framework/reconciler-suite.js @@ -560,9 +560,13 @@ export async function seedSyncthingApp(env, { // Seed the configurable test-app (controllable exit code / timed exit) and wait // for it to install. Returns { app, index, identifier }. Requires the test-app // binary to be built (bash test-infra/test-app/build.sh). -export async function seedTestApp(env, { name, exitCode = 0, exitAfterS = null } = {}) { +export async function seedTestApp(env, { + name, exitCode = 0, exitAfterS = null, logEveryMs = null, +} = {}) { await pushTestApp(name); - const app = await buildSeedableTestApp({ name, exitCode, exitAfterS }); + const app = await buildSeedableTestApp({ + name, exitCode, exitAfterS, logEveryMs, + }); const index = await seedAndInstall(env, app); return { app, index, identifier: `${name}_${name}` }; } diff --git a/test-infra/runner/framework/seed-helper.js b/test-infra/runner/framework/seed-helper.js index 9a8caff70b..e9a6f72880 100644 --- a/test-infra/runner/framework/seed-helper.js +++ b/test-infra/runner/framework/seed-helper.js @@ -397,7 +397,7 @@ export async function buildSeedableSyncthingApp({ * registry-helper.pushTestApp(name). */ export async function buildSeedableTestApp({ - name, exitCode = 0, exitAfterS = null, exitAfterMs = null, ...rest + name, exitCode = 0, exitAfterS = null, exitAfterMs = null, logEveryMs = null, ...rest }) { const environmentParameters = [`EXIT_CODE=${exitCode}`]; if (exitAfterS != null) environmentParameters.push(`EXIT_AFTER_S=${exitAfterS}`); @@ -405,6 +405,8 @@ export async function buildSeedableTestApp({ // own start still succeeds, so the node holds the app, but nothing that polls // docker afterwards can catch the process alive. if (exitAfterMs != null) environmentParameters.push(`EXIT_AFTER_MS=${exitAfterMs}`); + // a line on this interval, which docker writes to its data root as it comes + if (logEveryMs != null) environmentParameters.push(`LOG_EVERY_MS=${logEveryMs}`); const compose = [{ name, diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index e09fd5b769..0deedf10ef 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -709,11 +709,13 @@ function makeEnvShell(networkName) { // the volume (EPERM) and every run leaks its node volumes. Unmount to // expose the bare dirs and strip the flag while the node is still // running; containers going down makes this the last chance to exec. + // A docker data root a suite froze is thawed first: a node whose docker is + // held in the kernel cannot be stopped. await Promise.all(clients.map(async (client) => { if (!client?.container) return; await execInContainer( client.container, - 'for d in /mnt/appdata/flux-apps/*/; do umount -l "$d" 2>/dev/null; done; chattr -R -i /mnt/appdata/flux-apps 2>/dev/null; true', + 'fsfreeze -u /mnt/appdata/docker 2>/dev/null; for d in /mnt/appdata/flux-apps/*/; do umount -l "$d" 2>/dev/null; done; chattr -R -i /mnt/appdata/flux-apps 2>/dev/null; true', ).catch((e) => warn('immutable-flag sweep', e)); })); step(`immutable-sweep(${clients.filter(Boolean).length} nodes)`); @@ -970,7 +972,7 @@ export async function createTestEnv({ tickerAutostart = false, discoveryAutostart = false, nodeStatusOverrides = {}, rpcFailures = [], bootContext = 'running', initialHeight = DEFAULT_INITIAL_HEIGHT, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation = {}, locationTable = null, staticIp = true, policy = null, policySeeds = null, - awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], + awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], // systemd-mode nodes: the entrypoint execs a real systemd as PID 1, so dockerd // and fluxos run as units and FluxOS's output goes to the journal. Env-level // and uniform across nodes. @@ -986,6 +988,11 @@ export async function createTestEnv({ } // Arcane nodes whose OS starts syncthing paused, as a syncthing.service with // --paused does. A legacy node's syncthing is FluxOS's to start, always paused. + // Nodes whose docker keeps its data on a filesystem a suite can freeze + // (framework/host-stall.js), by node index. + for (const index of freezableDockerRoot) { + if (!Number.isInteger(index) || index < 0 || index >= nodes) throw new Error(`createTestEnv: freezableDockerRoot names node ${index}, outside a fleet of ${nodes}`); + } for (const index of pausedSyncthingNodes) { if (syncthing !== 'binary') throw new Error('createTestEnv: pausedSyncthingNodes needs syncthing: \'binary\''); if (legacyNodes.includes(index)) throw new Error(`createTestEnv: paused syncthing node ${index} is legacy, whose syncthing FluxOS starts`); @@ -1282,7 +1289,7 @@ export async function createTestEnv({ // mongo starts, i.e. inside the fleet boot, where the waits at risk are the // boot's own. await startInfraDeathWatch(env); - await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, systemdMode, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags); + await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, systemdMode, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot); return env; } catch (err) { // Boot failed: the env owns everything started so far. The shared teardown @@ -1311,7 +1318,7 @@ function mergeConfigs(base, override) { return result; } -async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], systemdMode = false, firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}) { +async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], systemdMode = false, firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = []) { // Everything built here registers onto the env shell as it comes up, so a // boot-phase throw leaves the partial state reachable (see makeEnvShell). const { @@ -1677,6 +1684,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod } if (!isLegacy) nodeEnv.FLUXOS_PATH = '/flux'; if (pausedSyncthingNodes.includes(i)) nodeEnv.FLUX_SYNCTHING_PAUSED = '1'; + if (freezableDockerRoot.includes(i)) nodeEnv.FLUX_FREEZABLE_DOCKER_ROOT = 'true'; if (syncthingFaults[i]) nodeEnv.FLUX_SYNCTHING_FAULT = syncthingFaults[i]; // WHETHER THIS NODE'S FLUXOS IS ROOT. Declared per node rather than derived from // legacy, because the field carries both: Arcane runs FluxOS as root, and an From 83029b3e06395375b56dbac617f8646a61e2c9a8 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:01:47 +0100 Subject: [PATCH 207/369] test(harness): 3302 holds a node whose database keeps timing out back from new placements Co-Authored-By: Claude Opus 5.5 --- ...t-keeps-timing-out-takes-on-no-new-apps.js | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js diff --git a/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js new file mode 100644 index 0000000000..7d6c867eb8 --- /dev/null +++ b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js @@ -0,0 +1,97 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { getAppContainerStatus } from '../framework/container.js'; +import { waitFor, waitForUp, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; +import { holdCommands, releaseCommands } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A mongod that keeps leaving FluxOS's calls unanswered, held with mongo-control as +// suite 3301 holds it. Once suspectRoundTrips round trips have run out within +// suspectWindowMs, the node holds itself back from new placements, and it lifts the +// hold suspectWindowMs after the last. The apps it runs stay: heavy load also makes +// calls time out. + +const CALL_TIMEOUT_MS = 5 * 1000; +const SUSPECT_WINDOW_MS = 60 * 1000; +// Long enough for many round trips to run out on every node. +const HOLD_MS = 45 * 1000; +const DATABASE_GAVE_NO_ANSWER = /a database round trip ran out/; +const HELD = /Placement hold set by databaseSuspect/; +const RELEASED = /Placement hold cleared by databaseSuspect/; + +function step(what) { + console.log(`# 3302 ${new Date().toISOString()} ${what}`); +} + +describe('a database that keeps timing out takes on no new apps', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2edbsuspect${Date.now()}`; + let appIdx; + let anchors; + let held = false; + let releasedAt; + + before(async function () { + this.timeout(420000 + HOLD_MS); + env = await createTestEnv({ + hookCtx: this, + nodes: 6, + tickerAutostart: false, + configOverrides: { database: { callTimeoutMs: CALL_TIMEOUT_MS, suspectWindowMs: SUSPECT_WINDOW_MS } }, + }); + await bootAndPeer(env); + ({ index: appIdx } = await seedSimpleApp(env, appName)); + await waitForUp(env.clients[appIdx], appName, 'the app is up'); + + anchors = env.clients.map((client) => client.getLastEventId()); + step('holding the commands'); + await holdCommands(HOLD_MS); + held = true; + try { + await new Promise((resolve) => { setTimeout(resolve, HOLD_MS); }); + } finally { + step('releasing the commands'); + await releaseCommands(); + held = false; + releasedAt = Date.now(); + step('released'); + } + }); + + after(async function () { + this.timeout(60000); + if (held) await releaseCommands().catch(() => {}); + await env?.teardown(); + }); + + it('counted round trips that ran out: the database did not answer FluxOS', function () { + const counted = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); + expect(counted, 'no node counted a round trip that ran out, so nothing below was tested').to.not.be.empty; + }); + + it('holds a node whose database kept timing out back from new placements', function () { + const holding = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, HELD)); + expect(holding, 'no node held itself back while its database left its calls unanswered').to.not.be.empty; + }); + + it('keeps the apps it runs', async function () { + this.timeout(30000); + await Promise.all(env.clients.map(async (client, i) => { + await assertNoEvent(client, 'app:removed', () => true, 1, { afterId: anchors[i] }); + })); + const running = await getAppContainerStatus(env.clients[appIdx].container, appName); + expect(running && running.status.startsWith('Up'), 'the app is still up').to.equal(true); + }); + + it('lifts the hold a window after the last round trip that ran out', async function () { + this.timeout(SUSPECT_WINDOW_MS + 120000); + const holding = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, HELD)); + await waitFor(() => holding.every((i) => env.nodeLogCount(i, RELEASED) >= env.nodeLogCount(i, HELD)), { + timeout: SUSPECT_WINDOW_MS + 90000, interval: 2000, label: 'every held node lifts its hold', + }); + expect(Date.now() - releasedAt, 'a hold lifted before its window could have passed').to.be.at.least(SUSPECT_WINDOW_MS - CALL_TIMEOUT_MS * 2); + }); +}); From 5aa0300190b10da4d1e78e7427d8f7ae3b2dbb8a Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:01:47 +0100 Subject: [PATCH 208/369] test(harness): 3303 takes a node whose docker the host stalls out of service, and kills its apps Co-Authored-By: Claude Opus 5.5 --- ...lled-host-takes-the-node-out-of-service.js | 173 ++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js new file mode 100644 index 0000000000..0dc47cd4ab --- /dev/null +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -0,0 +1,173 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, getAppContainerStatus } from '../framework/container.js'; +import { waitFor, waitForUp } from '../framework/wait.js'; +import { bootAndPeer, seedTestApp } from '../framework/reconciler-suite.js'; +import { freezeDocker, thawDocker, stuckThreads } from '../framework/host-stall.js'; +import { journalCount, restartFluxos } from '../framework/systemd-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A host that stops returning the disk under docker, on a real kernel: the +// filesystem the node's docker keeps its data on is frozen (framework/host-stall.js) +// while an app writes a log line every 200 ms, so docker's own threads are held in +// the kernel and never run - the stall FluxOS's detector reads from /proc. +// +// A thread held for one reading interval holds the node back from new placements, +// and the hold lifts when the thread runs again. Held for stuckIntervals intervals, +// the node goes out of service until it reboots, and removes every app: the normal +// removal, which needs docker and so does not return, runs out of time, and the +// app's processes are killed through its slice without docker. +// +// The nodes run systemd, so docker uses its systemd cgroup driver and every app's +// containers live in the app's own slice, as on a production node. + +const READ_INTERVAL_MS = 5 * 1000; +const STUCK_INTERVALS = 6; +const REMOVAL_DEADLINE_MS = 20 * 1000; +const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; + +const HELD = 'Placement hold set by hostStall'; +const RELEASED = 'Placement hold cleared by hostStall'; +const OUT_OF_SERVICE = 'Sticky DOS set by hostStall'; + +function step(what) { + console.log(`# 3303 ${new Date().toISOString()} ${what}`); +} + +const fluxosLog = (container, text) => journalCount(container, 'fluxos', text, { processOnly: true }); + +describe('a stalled host takes the node out of service', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2ehoststall${Date.now()}`; + let node; + let container; + let frozen = false; + + before(async function () { + this.timeout(480000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + tickerAutostart: false, + systemdMode: true, + freezableDockerRoot: [0, 1, 2, 3], + configOverrides: { + hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS }, + fluxapps: { appRemovalDeadlineMs: REMOVAL_DEADLINE_MS }, + }, + }); + await bootAndPeer(env); + ({ index: node } = await seedTestApp(env, { name: appName, logEveryMs: 200 })); + ({ container } = env.clients[node]); + await waitForUp(env.clients[node], appName, 'the app is up'); + }); + + after(async function () { + this.timeout(60000); + if (frozen) await thawDocker(container).catch(() => {}); + await env?.teardown(); + }); + + it('runs the app in its own slice', async function () { + const pid = (await execInContainer(container, `docker inspect -f '{{.State.Pid}}' flux${appName}_${appName}`)).stdout.trim(); + const cgroup = (await execInContainer(container, `cat /proc/${pid}/cgroup`)).stdout; + expect(cgroup, 'the app container is not in its slice, so the kill below has nothing to find').to.include(`/flux.slice/flux-${appName}.slice/`); + }); + + describe('a stall shorter than the threshold', function () { + let counts; + let stuck; + + before(async function () { + this.timeout(STALL_MS * 2); + counts = { held: await fluxosLog(container, HELD), released: await fluxosLog(container, RELEASED), out: await fluxosLog(container, OUT_OF_SERVICE) }; + step('freezing'); + await freezeDocker(container); + frozen = true; + try { + stuck = await stuckThreads(container, 'dockerd'); + await waitFor(async () => (await fluxosLog(container, HELD)) > counts.held, { + timeout: READ_INTERVAL_MS * 3, interval: 1000, label: 'the node holds itself back', + }); + } finally { + step('thawing'); + await thawDocker(container); + frozen = false; + } + }); + + it('held docker in the kernel', function () { + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + }); + + it('lifts the hold once docker runs again, and stays in service', async function () { + this.timeout(READ_INTERVAL_MS * 6); + await waitFor(async () => (await fluxosLog(container, RELEASED)) > counts.released, { + timeout: READ_INTERVAL_MS * 4, interval: 1000, label: 'the hold lifts', + }); + expect(await fluxosLog(container, OUT_OF_SERVICE), 'the node went out of service for a stall it recovered from').to.equal(counts.out); + const app = await getAppContainerStatus(container, appName); + expect(app && app.status.startsWith('Up'), 'the app is still up').to.equal(true); + }); + }); + + describe('a stall past the threshold', function () { + let stuck; + + before(async function () { + this.timeout(STALL_MS * 4); + const out = await fluxosLog(container, OUT_OF_SERVICE); + step('freezing'); + await freezeDocker(container); + frozen = true; + stuck = await stuckThreads(container, 'dockerd'); + await waitFor(async () => (await fluxosLog(container, OUT_OF_SERVICE)) > out, { + timeout: STALL_MS + READ_INTERVAL_MS * 4, interval: 2000, label: 'the node goes out of service', + }); + step('out of service'); + }); + + it('held docker in the kernel', function () { + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + }); + + it('takes the node out of service, naming the stall', async function () { + const { data } = await env.clients[node].getDOSState(); + expect(data.dosState).to.be.at.least(100); + expect(data.dosMessage).to.include('Host stall').and.include('dockerd'); + }); + + it('kills the app its normal removal could not remove, and it can never run again', async function () { + this.timeout(REMOVAL_DEADLINE_MS * 4); + await waitFor(async () => (await fluxosLog(container, `${appName} can no longer run`)) > 0, { + timeout: REMOVAL_DEADLINE_MS * 3, interval: 2000, label: 'the app can no longer run', + }); + expect(await fluxosLog(container, `${appName} still runs after its removal`), 'the app was killed without its normal removal running out first').to.be.above(0); + const procs = (await execInContainer(container, `cat /sys/fs/cgroup/flux.slice/flux-${appName}.slice/*/cgroup.procs 2>/dev/null; true`)).stdout.trim(); + expect(procs, 'processes of the app are left in its slice').to.equal(''); + }); + + it('stays out of service through a FluxOS restart in the same boot', async function () { + this.timeout(300000); + const restored = await fluxosLog(container, OUT_OF_SERVICE); + step('restarting FluxOS'); + await restartFluxos(container, { timeout: 120000, readyTimeoutMs: 180000 }); + await waitFor(async () => (await fluxosLog(container, OUT_OF_SERVICE)) > restored, { + timeout: 60000, interval: 2000, label: 'the restarted node restores its trip', + }); + const { data } = await env.clients[node].getDOSState(); + expect(data.dosState, 'a FluxOS restart returned the node to service').to.be.at.least(100); + }); + + it('stays out of service once docker runs again', async function () { + this.timeout(READ_INTERVAL_MS * 6); + await thawDocker(container); + frozen = false; + await new Promise((resolve) => { setTimeout(resolve, READ_INTERVAL_MS * 3); }); + const { data } = await env.clients[node].getDOSState(); + expect(data.dosState, 'the node returned to service without a reboot').to.be.at.least(100); + }); + }); +}); From 03213f2a9ac7c6e6ddd869c622c208e28d0bde2b Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:23:30 +0100 Subject: [PATCH 209/369] fix(apps): the kill's check reads every cgroup under the app's slice A process lives only in a leaf cgroup, so an app's processes are in its containers' scopes below the slice, and the slice's own cgroup.procs is empty. The check now reads the slice and every cgroup below it. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 17 +++++++++++-- tests/unit/appKill.test.js | 25 +++++++++++++++----- 2 files changed, 34 insertions(+), 8 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index aa0b2a0f78..49cbf2ebb5 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -144,6 +144,20 @@ async function killPendingOrGone(pid) { }); } +/** + * Every process in a cgroup and the cgroups below it. A process lives only in a + * leaf, so an app's processes are in the scopes under its slice, not the slice. + * @param {string} dir + * @returns {Promise} Their pids. + */ +async function processesUnder(dir) { + const own = (await fs.readFile(path.join(dir, 'cgroup.procs'), 'utf8').catch(() => '')) + .split('\n').map((line) => line.trim()).filter(Boolean); + const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []); + const below = await Promise.all(entries.filter((entry) => entry.isDirectory()).map((entry) => processesUnder(path.join(dir, entry.name)))); + return [...own, ...below.flat()]; +} + /** * Which of the app's processes could still run its code: every one that is * neither gone nor has SIGKILL pending. @@ -152,8 +166,7 @@ async function killPendingOrGone(pid) { */ async function stillRunnable(appName) { const cgroups = await cgroupsOf(appName); - const pids = (await Promise.all(cgroups.map((dir) => fs.readFile(path.join(dir, 'cgroup.procs'), 'utf8').catch(() => '')))) - .flatMap((text) => text.split('\n').map((line) => line.trim()).filter(Boolean)); + const pids = (await Promise.all(cgroups.map(processesUnder))).flat(); const settled = await Promise.all(pids.map(killPendingOrGone)); return pids.filter((pid, i) => !settled[i]); } diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index a964c33980..8fe8bcce23 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -27,6 +27,13 @@ describe('appKill tests', () => { fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pids[0] ?? 0) } })); return cgroup(path.join('system.slice', `docker-${id(n)}.scope`), pids); } + // An app's slice, as the kernel lays it out: no process of its own, and each + // container's processes in the container's scope below it. + function slice(unit, scopes) { + const dir = cgroup(path.join('flux.slice', unit), []); + Object.entries(scopes).forEach(([scope, pids]) => cgroup(path.join('flux.slice', unit, scope), pids)); + return dir; + } function processWith(pid, { sigPnd = '0000000000000000', shdPnd = '0000000000000000' } = {}) { fs.mkdirSync(path.join(procRoot, pid), { recursive: true }); fs.writeFileSync(path.join(procRoot, pid, 'status'), `Name:\tapp\nSigPnd:\t${sigPnd}\nShdPnd:\t${shdPnd}\n`); @@ -68,17 +75,17 @@ describe('appKill tests', () => { describe('kill', () => { it('kills the app\'s slice and the scopes of its containers made before slices, and no other app\'s', async () => { - const slice = cgroup(path.join('flux.slice', 'flux-my\\x2dapp.slice'), ['10']); + const appSliceDir = slice('flux-my\\x2dapp.slice', { [`docker-${id(9)}.scope`]: ['10'] }); const web = container(1, 'fluxweb_my-app', ['11']); const single = container(2, 'fluxmy-app', ['12']); container(3, 'fluxweb_my-app2', ['13']); container(4, 'fluxweb_other', ['14']); - cgroup(path.join('flux.slice', 'flux-my.slice'), ['15']); + slice('flux-my.slice', { [`docker-${id(8)}.scope`]: ['15'] }); const sent = await appKill.kill('my-app'); - expect(sent).to.have.members([slice, web, single]); - expect(killed).to.have.members([slice, web, single]); + expect(sent).to.have.members([appSliceDir, web, single]); + expect(killed).to.have.members([appSliceDir, web, single]); }); it('sends nothing for an app with no cgroup left', async () => { @@ -100,7 +107,7 @@ describe('appKill tests', () => { describe('stillRunnable', () => { beforeEach(() => { - cgroup(path.join('flux.slice', 'flux-app.slice'), ['21', '22', '23', '24']); + slice('flux-app.slice', { [`docker-${id(1)}.scope`]: ['21', '22'], [`docker-${id(2)}.scope`]: ['23', '24'] }); }); it('names each process that is neither gone nor has the kill pending', async () => { @@ -118,6 +125,12 @@ describe('appKill tests', () => { expect(await appKill.stillRunnable('app')).to.deep.equal(['21', '22', '23', '24']); }); + it('reads the processes in every scope under the slice, the slice itself holding none', async () => { + ['21', '22', '23', '24'].forEach((pid) => processWith(pid)); + + expect(await appKill.stillRunnable('app')).to.have.members(['21', '22', '23', '24']); + }); + it('names nothing for an app with no cgroup left', async () => { expect(await appKill.stillRunnable('other')).to.deep.equal([]); }); @@ -125,7 +138,7 @@ describe('appKill tests', () => { describe('appsOnDisk', () => { it('names every app with a slice or a container record, and no container that is not an app\'s', async () => { - cgroup(path.join('flux.slice', 'flux-sliced\\x2dapp.slice'), []); + slice('flux-sliced\\x2dapp.slice', {}); container(1, 'fluxweb_composed', ['11']); container(2, 'fluxsingle', ['12']); container(3, 'watchtower', ['13']); From 6cf6f37222051b7155acc5ed4d68424bbadd6173 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:25:08 +0100 Subject: [PATCH 210/369] fix(apps): an app whose removal is still running is not removed again A removal past its deadline keeps running, so a second removal of the same app would only wait behind the same stall, and each DOS pass would add another. The removal is skipped while one is running, and a host stall's unload still checks and kills the app. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/nodeStatusMonitor.js | 28 +++++++++++++++++-- tests/unit/nodeStatusMonitor.test.js | 21 ++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 422866541e..5cdacd1e49 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -21,6 +21,28 @@ let unloadingForHostStall = false; let deps = null; const REMOVAL_DEADLINE_MS = config.fluxapps.appRemovalDeadlineMs; +const STILL_REMOVING = 'stillRemoving'; +/** Apps whose removal started here and has not returned, deadline or not. */ +const removalsRunning = new Set(); + +/** + * Remove one app, waiting for it at most REMOVAL_DEADLINE_MS. A removal past + * its deadline keeps running, so an app whose removal is still running is not + * removed again: a second removal beside it would only wait behind the same + * stall, and the next pass would add a third. + * @param {Function} removeAppLocallyFn + * @param {string} name + * @param {boolean} canBroadcast + * @returns {Promise} The removal's outcome, TIMED_OUT, or STILL_REMOVING. + */ +async function removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) { + if (removalsRunning.has(name)) return STILL_REMOVING; + removalsRunning.add(name); + const removal = Promise.resolve() + .then(() => removeAppLocallyFn(name, null, true, false, canBroadcast)) + .finally(() => removalsRunning.delete(name)); + return withinDeadline(removal, REMOVAL_DEADLINE_MS); +} // Database collections const globalAppsLocations = config.database.appsglobal.collections.appsLocations; @@ -49,9 +71,11 @@ async function removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason) log.warn(`monitorNodeStatus - Removing application ${installedApp.name} locally`); // One removal that never returns must not hold every other app here. // eslint-disable-next-line no-await-in-loop - const outcome = await withinDeadline(removeAppLocallyFn(installedApp.name, null, true, false, canBroadcast), REMOVAL_DEADLINE_MS); + const outcome = await removeWithinDeadline(removeAppLocallyFn, installedApp.name, canBroadcast); if (outcome === TIMED_OUT) { log.error(`monitorNodeStatus - Application ${installedApp.name} was not removed within ${REMOVAL_DEADLINE_MS / 1000}s, moving on`); + } else if (outcome === STILL_REMOVING) { + log.warn(`monitorNodeStatus - Application ${installedApp.name} is still being removed, moving on`); } else { log.warn(`monitorNodeStatus - Application ${installedApp.name} locally removed`); } @@ -104,7 +128,7 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { for (const name of names) { log.warn(`monitorNodeStatus - Application ${name} going to be removed: host stall`); // eslint-disable-next-line no-await-in-loop - const outcome = await withinDeadline(removeAppLocallyFn(name, null, true, false, canBroadcast), REMOVAL_DEADLINE_MS) + const outcome = await removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) .catch(() => RemovalOutcome.FAILED); // eslint-disable-next-line no-await-in-loop if ((await appKill.stillRunnable(name)).length) { diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index f222aa4ae7..e29293ac02 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -219,6 +219,27 @@ describe('nodeStatusMonitor tests', () => { sinon.assert.calledOnce(remove); }); + it('starts no second removal of an app whose removal is still running, and still checks it', async () => { + let finish; + const remove = sinon.stub().callsFake(() => new Promise((resolve) => { finish = resolve; })); + const first = monitor.unloadForHostStall(installed('stuck'), remove); + await clock.tickAsync(DEADLINE); + await first; + appKill.stillRunnable.resetHistory(); + + await monitor.unloadForHostStall(installed('stuck'), remove); + + sinon.assert.calledOnce(remove); + sinon.assert.calledWith(appKill.stillRunnable, 'stuck'); + finish('removed'); + await clock.tickAsync(0); + const third = monitor.unloadForHostStall(installed('stuck'), remove); + await clock.tickAsync(0); + sinon.assert.calledTwice(remove); + finish('removed'); + await third; + }); + it('moves past a DOS removal that does not return within the deadline', async () => { dos = true; const remove = sinon.stub(); From c1a4296520886d0c19063a3c9952083a34952316 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:25:43 +0100 Subject: [PATCH 211/369] test(harness): a failure dump thaws a frozen node's docker before it reads the node The dump and the teardown both reach a node through its docker, which cannot answer while a suite holds it frozen; thawEveryNode runs first in each. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/host-stall.js | 13 +++++++++++++ test-infra/runner/framework/log-on-failure.js | 4 ++++ test-infra/runner/framework/test-env.js | 7 ++++--- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index 390de7686a..c02359c702 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -27,6 +27,19 @@ export async function thawDocker(container) { await run(container, `fsfreeze -u ${DOCKER_ROOT}`, 'thawDocker'); } +/** + * Thaw every node's docker that is frozen, and pass over the rest: the last step + * before anything reads a node through its docker or stops it, since a node whose + * docker is held in the kernel can be neither read nor stopped. Never rejects. + * @param {Array<{container?: object}>} clients + */ +export async function thawEveryNode(clients) { + await Promise.all(clients.map(async (client) => { + if (!client?.container) return; + await execInContainer(client.container, `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; true`).catch(() => {}); + })); +} + /** * One reading of every thread of the processes named `comm` in the node: its * state, its voluntary plus nonvoluntary context switches, and where it waits. diff --git a/test-infra/runner/framework/log-on-failure.js b/test-infra/runner/framework/log-on-failure.js index 3537baee14..a418d17aeb 100644 --- a/test-infra/runner/framework/log-on-failure.js +++ b/test-infra/runner/framework/log-on-failure.js @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { activeTestEnvs } from './test-env.js'; import { execInContainer } from './container.js'; import { SYNCTHING_HOME } from './syncthing-real.js'; +import { thawEveryNode } from './host-stall.js'; import failedHooksModule from './failed-hooks.cjs'; const { failedHooks } = failedHooksModule; @@ -155,6 +156,9 @@ export function dumpLogsOnFailure(getEnv) { // daemon, so they are fetched up front and written alongside them. Each entry // is best-effort (see env.infraDiagnostics) — a log fetch that failed carries // an `error` and is reported, never thrown. + // Everything below reads the nodes through their docker, which a suite may + // have left frozen. + await Promise.all(envs.map((env) => thawEveryNode(env.clients ?? []))); const infraByEnv = await Promise.all(envs.map((env) => env.infraDiagnostics())); const cgroupsByEnv = await Promise.all(envs.map((env) => cgroupState(env).catch( (err) => `cgroup probe failed: ${err.message}\n`, diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 0deedf10ef..3d1c9d7d86 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -15,6 +15,7 @@ import { fileURLToPath } from 'node:url'; import http from 'node:http'; import { nodeClient } from './node-client.js'; import { execInContainer, restartFluxos } from './container.js'; +import { thawEveryNode } from './host-stall.js'; import { waitFor } from './wait.js'; import { HttpPollWaitStrategy } from './http-wait-strategy.js'; import { TcpPollWaitStrategy } from './tcp-wait-strategy.js'; @@ -709,13 +710,13 @@ function makeEnvShell(networkName) { // the volume (EPERM) and every run leaks its node volumes. Unmount to // expose the bare dirs and strip the flag while the node is still // running; containers going down makes this the last chance to exec. - // A docker data root a suite froze is thawed first: a node whose docker is - // held in the kernel cannot be stopped. + await thawEveryNode(clients); + step('thaw'); await Promise.all(clients.map(async (client) => { if (!client?.container) return; await execInContainer( client.container, - 'fsfreeze -u /mnt/appdata/docker 2>/dev/null; for d in /mnt/appdata/flux-apps/*/; do umount -l "$d" 2>/dev/null; done; chattr -R -i /mnt/appdata/flux-apps 2>/dev/null; true', + 'for d in /mnt/appdata/flux-apps/*/; do umount -l "$d" 2>/dev/null; done; chattr -R -i /mnt/appdata/flux-apps 2>/dev/null; true', ).catch((e) => warn('immutable-flag sweep', e)); })); step(`immutable-sweep(${clients.filter(Boolean).length} nodes)`); From ca6ac33f89963d4ac62731539d76f0e6582b532d Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:42:11 +0100 Subject: [PATCH 212/369] fix(apps): a host stall's removal says an app it could not remove can no longer run A removal that runs out often stops the app first: docker delivers its stop signal, which needs no disk, and hangs on its own records afterwards. When the check then finds nothing of the app able to run, the node says so, and an operator sees the end state whichever step reached it. Co-Authored-By: Claude Opus 5.5 --- .../services/appMonitoring/nodeStatusMonitor.js | 2 ++ tests/unit/nodeStatusMonitor.test.js | 17 +++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 5cdacd1e49..5aeaa98c12 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -142,6 +142,8 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { } else { log.warn(`monitorNodeStatus - Application ${name} can no longer run`); } + } else if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + log.warn(`monitorNodeStatus - Application ${name} was not removed (${outcome}), and can no longer run`); } } } finally { diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index e29293ac02..c90776ba82 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -184,6 +184,23 @@ describe('nodeStatusMonitor tests', () => { sinon.assert.calledWithMatch(logStub.warn, /stuck can no longer run/); }); + it('says an app its removal could not finish can no longer run, when nothing of it is left to kill', async () => { + const remove = sinon.stub().callsFake(never); + + const unload = monitor.unloadForHostStall(installed('stopped'), remove); + await clock.tickAsync(DEADLINE); + await unload; + + sinon.assert.notCalled(appKill.kill); + sinon.assert.calledWithMatch(logStub.warn, /stopped was not removed \(timedOut\), and can no longer run/); + }); + + it('says nothing more of an app its removal removed', async () => { + await monitor.unloadForHostStall(installed('gone'), sinon.stub().resolves('removed')); + + sinon.assert.neverCalledWithMatch(logStub.warn, /can no longer run/); + }); + it('kills an app whose removal failed while its processes can still run', async () => { const remove = sinon.stub().resolves('failed'); appKill.stillRunnable.onFirstCall().resolves(['41']).onSecondCall().resolves([]); From d866afe0a89e4a46a62fb5ae16f0da6afd16487e Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:42:46 +0100 Subject: [PATCH 213/369] test(harness): 3302 and 3303 assert what is true on any tree before what the fix makes true 3302's canary is the driver's own timeout message, which a node logs with or without the fix, and its last case requires a hold before it times one. 3303 asserts the app's end state: its removal ran out, and no process left in its slice can run, read inside the node independently of FluxOS (host-stall.processesThatCanRun). Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/host-stall.js | 17 +++++++++++++++++ ...at-keeps-timing-out-takes-on-no-new-apps.js | 11 +++++++---- ...alled-host-takes-the-node-out-of-service.js | 18 +++++++++++------- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index c02359c702..7cdeee0715 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -40,6 +40,23 @@ export async function thawEveryNode(clients) { })); } +/** + * The processes in a cgroup and every cgroup below it that could still run: + * neither gone nor with SIGKILL pending. Read inside the node, independently of + * FluxOS. + * @param {object} container + * @param {string} cgroupDir Under /sys/fs/cgroup. + * @returns {Promise} Their pids; empty when nothing in it can run again. + */ +export async function processesThatCanRun(container, cgroupDir) { + const script = `[ -d '${cgroupDir}' ] || exit 0; for p in $(find '${cgroupDir}' -name cgroup.procs -exec cat {} +); do ` + + 's=/proc/$p/status; [ -r $s ] || continue; ' + + 'k=$(awk \'/^(SigPnd|ShdPnd):/{print $2}\' $s | while read m; do [ $(( 0x$m & 0x100 )) -ne 0 ] && echo y; done); ' + + '[ -z "$k" ] && echo $p; done; true'; + const out = await run(container, script, `reading the processes under ${cgroupDir}`); + return out.split('\n').map((line) => line.trim()).filter(Boolean); +} + /** * One reading of every thread of the processes named `comm` in the node: its * state, its voluntary plus nonvoluntary context switches, and where it waits. diff --git a/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js index 7d6c867eb8..dc14dbb5ac 100644 --- a/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js +++ b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js @@ -17,7 +17,9 @@ const CALL_TIMEOUT_MS = 5 * 1000; const SUSPECT_WINDOW_MS = 60 * 1000; // Long enough for many round trips to run out on every node. const HOLD_MS = 45 * 1000; -const DATABASE_GAVE_NO_ANSWER = /a database round trip ran out/; +// The driver's own words for a round trip and for a pooled-connection wait that ran +// out, as suite 3301 reads them: the canary that the calls failed, on any tree. +const DATABASE_GAVE_NO_ANSWER = /connection \d+ to \S+ timed out|Timed out while checking out a connection from connection pool/; const HELD = /Placement hold set by databaseSuspect/; const RELEASED = /Placement hold cleared by databaseSuspect/; @@ -67,9 +69,9 @@ describe('a database that keeps timing out takes on no new apps', function () { await env?.teardown(); }); - it('counted round trips that ran out: the database did not answer FluxOS', function () { - const counted = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); - expect(counted, 'no node counted a round trip that ran out, so nothing below was tested').to.not.be.empty; + it('reached FluxOS: a node logged a call the database did not answer', function () { + const stalled = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); + expect(stalled, 'no node logged a database call that timed out, so nothing below was tested').to.not.be.empty; }); it('holds a node whose database kept timing out back from new placements', function () { @@ -89,6 +91,7 @@ describe('a database that keeps timing out takes on no new apps', function () { it('lifts the hold a window after the last round trip that ran out', async function () { this.timeout(SUSPECT_WINDOW_MS + 120000); const holding = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, HELD)); + expect(holding, 'no node held itself back, so there is no hold to lift').to.not.be.empty; await waitFor(() => holding.every((i) => env.nodeLogCount(i, RELEASED) >= env.nodeLogCount(i, HELD)), { timeout: SUSPECT_WINDOW_MS + 90000, interval: 2000, label: 'every held node lifts its hold', }); diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 0dc47cd4ab..06c16a9eeb 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -4,7 +4,9 @@ import { createTestEnv } from '../framework/test-env.js'; import { execInContainer, getAppContainerStatus } from '../framework/container.js'; import { waitFor, waitForUp } from '../framework/wait.js'; import { bootAndPeer, seedTestApp } from '../framework/reconciler-suite.js'; -import { freezeDocker, thawDocker, stuckThreads } from '../framework/host-stall.js'; +import { + freezeDocker, thawDocker, stuckThreads, processesThatCanRun, +} from '../framework/host-stall.js'; import { journalCount, restartFluxos } from '../framework/systemd-control.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -139,14 +141,16 @@ describe('a stalled host takes the node out of service', function () { expect(data.dosMessage).to.include('Host stall').and.include('dockerd'); }); - it('kills the app its normal removal could not remove, and it can never run again', async function () { + it('leaves nothing of the app that can run, though its removal could not finish', async function () { this.timeout(REMOVAL_DEADLINE_MS * 4); - await waitFor(async () => (await fluxosLog(container, `${appName} can no longer run`)) > 0, { - timeout: REMOVAL_DEADLINE_MS * 3, interval: 2000, label: 'the app can no longer run', + await waitFor(async () => (await fluxosLog(container, `${appName} can no longer run`)) > 0 + || (await fluxosLog(container, `${appName} could not be stopped`)) > 0, { + timeout: REMOVAL_DEADLINE_MS * 3, interval: 2000, label: 'the node says what is left of the app', }); - expect(await fluxosLog(container, `${appName} still runs after its removal`), 'the app was killed without its normal removal running out first').to.be.above(0); - const procs = (await execInContainer(container, `cat /sys/fs/cgroup/flux.slice/flux-${appName}.slice/*/cgroup.procs 2>/dev/null; true`)).stdout.trim(); - expect(procs, 'processes of the app are left in its slice').to.equal(''); + const ranOut = (await fluxosLog(container, `${appName} was not removed`)) + (await fluxosLog(container, `${appName} still runs after its removal`)); + expect(ranOut, 'the app\'s removal finished, so its stall was never what removed it').to.be.above(0); + expect(await fluxosLog(container, `${appName} could not be stopped`), 'the node could not stop the app').to.equal(0); + expect(await processesThatCanRun(container, `/sys/fs/cgroup/flux.slice/flux-${appName}.slice`), 'a process of the app can still run').to.deep.equal([]); }); it('stays out of service through a FluxOS restart in the same boot', async function () { From b6fb6b58817859b1946c999c109bb74f565bb9a5 Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 17 Oct 2026 12:49:55 +0100 Subject: [PATCH 214/369] test(harness): 3303 reads the node's words for an app its removal could not finish An app stopped before its removal ran out is reported as 'was not removed (timedOut), and can no longer run', which the substring ' can no longer run' does not match. Co-Authored-By: Claude Opus 5.5 --- .../tests/3303-a-stalled-host-takes-the-node-out-of-service.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 06c16a9eeb..cb95fddb65 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -143,7 +143,10 @@ describe('a stalled host takes the node out of service', function () { it('leaves nothing of the app that can run, though its removal could not finish', async function () { this.timeout(REMOVAL_DEADLINE_MS * 4); + // the node's two words for an app with nothing left that can run - killed, or + // stopped before its removal ran out - and its one for an app it could not stop await waitFor(async () => (await fluxosLog(container, `${appName} can no longer run`)) > 0 + || (await fluxosLog(container, `${appName} was not removed`)) > 0 || (await fluxosLog(container, `${appName} could not be stopped`)) > 0, { timeout: REMOVAL_DEADLINE_MS * 3, interval: 2000, label: 'the node says what is left of the app', }); From 86217e74771e888653382fbd792dde506984afbe Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 07:06:52 +0100 Subject: [PATCH 215/369] fix(apps): the test event stream carries every placement hold and each app a host stall unloads placementHold:set and placementHold:cleared name the owner and the reason. hostStall:appUnloaded names the app, its removal's outcome, whether it was killed, and the processes of it that can still run. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/nodeStatusMonitor.js | 15 ++++++++---- ZelBack/src/services/fluxNetworkHelper.js | 2 ++ tests/unit/fluxNetworkHelper.test.js | 23 +++++++++++++++++++ tests/unit/nodeStatusMonitor.test.js | 23 +++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 5aeaa98c12..f78d2c503e 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -12,6 +12,7 @@ const hostStallService = require('../hostStallService'); const appKill = require('../appLifecycle/appKill'); const { RemovalOutcome } = require('../utils/removalOutcome'); const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); +const fluxEventBus = require('../utils/fluxEventBus'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -131,20 +132,26 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { const outcome = await removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) .catch(() => RemovalOutcome.FAILED); // eslint-disable-next-line no-await-in-loop - if ((await appKill.stillRunnable(name)).length) { + let runnable = await appKill.stillRunnable(name); + const killed = runnable.length > 0; + if (killed) { log.error(`monitorNodeStatus - Application ${name} still runs after its removal (${outcome}), killing it`); // eslint-disable-next-line no-await-in-loop await appKill.kill(name); // eslint-disable-next-line no-await-in-loop - const left = await appKill.stillRunnable(name); - if (left.length) { - log.error(`monitorNodeStatus - Application ${name} could not be stopped: processes ${left.join(', ')} can still run`); + runnable = await appKill.stillRunnable(name); + if (runnable.length) { + log.error(`monitorNodeStatus - Application ${name} could not be stopped: processes ${runnable.join(', ')} can still run`); } else { log.warn(`monitorNodeStatus - Application ${name} can no longer run`); } } else if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { log.warn(`monitorNodeStatus - Application ${name} was not removed (${outcome}), and can no longer run`); } + // runnable: the processes of the app that can still run its code, empty when none can + fluxEventBus.publish('hostStall:appUnloaded', { + app: name, removal: outcome, killed, runnable, + }); } } finally { unloadingForHostStall = false; diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 16818e2d1b..73b1000424 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -1189,6 +1189,7 @@ function setPlacementHold(owner, reason) { if (placementHolds.get(owner) === reason) return; placementHolds.set(owner, reason); log.info(`Placement hold set by ${owner}: ${reason}`); + fluxEventBus.publish('placementHold:set', { owner, reason }); } /** @@ -1202,6 +1203,7 @@ function clearPlacementHold(owner) { if (reason === undefined) return; placementHolds.delete(owner); log.info(`Placement hold cleared by ${owner} (was: ${reason})`); + fluxEventBus.publish('placementHold:cleared', { owner, reason }); } /** diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 42855f5de1..06e30ac7de 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -4004,6 +4004,29 @@ describe('fluxNetworkHelper tests', () => { expect(fluxNetworkHelper.isPlacementHeld()).to.equal(false); }); + it('publishes a hold when it is set, and again when its reason changes', () => { + const publish = sinon.stub(fluxEventBus, 'publish'); + + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'first'); + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'first'); + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'second'); + + expect(publish.args.filter(([name]) => name === 'placementHold:set')).to.deep.equal([ + ['placementHold:set', { owner: RESIDENTIAL_DOS, reason: 'first' }], + ['placementHold:set', { owner: RESIDENTIAL_DOS, reason: 'second' }], + ]); + }); + + it('publishes a release only of a hold that stood', () => { + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'held'); + const publish = sinon.stub(fluxEventBus, 'publish'); + + fluxNetworkHelper.clearPlacementHold(RESIDENTIAL_DOS); + fluxNetworkHelper.clearPlacementHold(RESIDENTIAL_DOS); + + sinon.assert.calledOnceWithExactly(publish, 'placementHold:cleared', { owner: RESIDENTIAL_DOS, reason: 'held' }); + }); + it('refuses an owner it does not know, rather than minting one', () => { // An unknown owner is a caller that was never given an identity. Accepted, // it would hold the node under a name no release path knows about. diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index c90776ba82..974661ddad 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -123,12 +123,14 @@ describe('nodeStatusMonitor tests', () => { let appKill; let onTrip; let dos; + let bus; const never = () => new Promise(() => {}); const installed = (...names) => sinon.stub().resolves({ status: 'success', data: names.map((name) => ({ name })) }); beforeEach(() => { clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); dos = false; + bus = { publish: sinon.stub() }; appKill = { appsOnDisk: sinon.stub().resolves([]), stillRunnable: sinon.stub().resolves([]), @@ -148,6 +150,7 @@ describe('nodeStatusMonitor tests', () => { }, '../hostStallService': { onTrip: (listener) => { onTrip = listener; } }, '../appLifecycle/appKill': appKill, + '../utils/fluxEventBus': bus, '../../lib/log': logStub, }); }); @@ -156,6 +159,26 @@ describe('nodeStatusMonitor tests', () => { clock.restore(); }); + it('publishes an app its removal removed, with nothing left to run', async () => { + await monitor.unloadForHostStall(installed('gone'), sinon.stub().resolves('removed')); + + sinon.assert.calledOnceWithExactly(bus.publish, 'hostStall:appUnloaded', { + app: 'gone', removal: 'removed', killed: false, runnable: [], + }); + }); + + it('publishes an app it killed after its removal ran out, and what of it can still run', async () => { + appKill.stillRunnable.onFirstCall().resolves(['41', '42']).onSecondCall().resolves(['42']); + + const unload = monitor.unloadForHostStall(installed('stuck'), sinon.stub().callsFake(never)); + await clock.tickAsync(DEADLINE); + await unload; + + sinon.assert.calledOnceWithExactly(bus.publish, 'hostStall:appUnloaded', { + app: 'stuck', removal: 'timedOut', killed: true, runnable: ['42'], + }); + }); + it('removes every app once the host stalls, starting from the trip', async () => { const remove = sinon.stub().resolves('removed'); monitor.initialize(installed('one', 'two'), remove); From 129e41d1841f4bf4896352fe4d59569ce01824e1 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 07:11:12 +0100 Subject: [PATCH 216/369] test(harness): a suite can freeze any systemd unit of a node, and mongo-control says whose commands it held systemd-control.freezeUnit and thawUnit stop a unit's every process where it stands, through its cgroup's freeze switch, and host-stall.thawEveryNode thaws every frozen cgroup with the docker data root. releaseCommands resolves to how many commands mongod held from each client address. The client hears placementHold:set, placementHold:cleared and hostStall:appUnloaded. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/host-stall.js | 11 ++++--- test-infra/runner/framework/mongo-control.js | 11 ++++++- test-infra/runner/framework/node-client.js | 3 ++ .../runner/framework/systemd-control.js | 32 ++++++++++++++++++- 4 files changed, 51 insertions(+), 6 deletions(-) diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index 7cdeee0715..2c94af09f4 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -28,15 +28,18 @@ export async function thawDocker(container) { } /** - * Thaw every node's docker that is frozen, and pass over the rest: the last step - * before anything reads a node through its docker or stops it, since a node whose - * docker is held in the kernel can be neither read nor stopped. Never rejects. + * Thaw every node's docker data root that is frozen, and every cgroup frozen in it + * (systemd-control.freezeUnit), and pass over the rest: the last step before + * anything reads a node through its docker or stops it, since a node whose docker + * is held can be neither read nor stopped. Never rejects. * @param {Array<{container?: object}>} clients */ export async function thawEveryNode(clients) { + const thaw = `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; ` + + 'for f in $(find /sys/fs/cgroup -name cgroup.freeze 2>/dev/null); do [ "$(cat $f)" = 1 ] && echo 0 > $f; done; true'; await Promise.all(clients.map(async (client) => { if (!client?.container) return; - await execInContainer(client.container, `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; true`).catch(() => {}); + await execInContainer(client.container, thaw).catch(() => {}); })); } diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js index cd0afae474..a7ca8b6d90 100644 --- a/test-infra/runner/framework/mongo-control.js +++ b/test-infra/runner/framework/mongo-control.js @@ -38,20 +38,29 @@ export async function holdCommands(holdMs) { // Turning the failpoint off waits for every command inside it to leave, and a held // command stays for the whole hold. So the held commands are killed until the // release completes: their wait is interruptible, and they leave at once. +// +// Resolves to how many commands were held from each client address (ip, no port): +// read on the server, so it says the calls reached mongod and waited, whatever the +// client did about it. export async function releaseCommands() { - await withAdmin(async (admin) => { + return withAdmin(async (admin) => { + const heldFrom = new Map(); let released = false; const off = admin.command({ configureFailPoint: 'failCommand', mode: 'off' }).finally(() => { released = true; }); while (!released) { // eslint-disable-next-line no-await-in-loop const { inprog } = await admin.command({ currentOp: 1, active: true }); const held = inprog.filter((op) => op.command && HELD_COMMANDS.some((name) => name in op.command)); + for (const op of held) heldFrom.set(op.opid, (op.client ?? '').replace(/:\d+$/, '')); // eslint-disable-next-line no-await-in-loop await Promise.all(held.map((op) => admin.command({ killOp: 1, op: op.opid }).catch(() => {}))); // eslint-disable-next-line no-await-in-loop await Promise.race([off.catch(() => {}), new Promise((resolve) => { setTimeout(resolve, 1000); })]); } await off; + const byClient = {}; + for (const ip of heldFrom.values()) byClient[ip] = (byClient[ip] ?? 0) + 1; + return byClient; }); } diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index de06d8e365..61d985f67c 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -310,6 +310,9 @@ export function nodeClient(nodeNum) { 'spawner:networkErrorSkip', 'spawner:paused', 'spawner:resumed', + 'placementHold:set', + 'placementHold:cleared', + 'hostStall:appUnloaded', 'network:apprunning', 'network:appinstalling', 'network:appinstallingerror', diff --git a/test-infra/runner/framework/systemd-control.js b/test-infra/runner/framework/systemd-control.js index f466a16934..1baae73288 100644 --- a/test-infra/runner/framework/systemd-control.js +++ b/test-infra/runner/framework/systemd-control.js @@ -12,7 +12,7 @@ import { execInContainer } from './container.js'; import { waitFor } from './wait.js'; const FLUXOS_UNIT = 'fluxos'; -const DOCKERD_UNIT = 'dockerd'; +export const DOCKERD_UNIT = 'dockerd'; // Single-quote for `sh -c`: end the quote, escape the quote, reopen. const shQuote = (s) => `'${String(s).replace(/'/g, "'\\''")}'`; @@ -105,6 +105,36 @@ export async function resumeDockerd(container, opts = {}) { { timeout: opts.timeout ?? 60000, interval: 500, label: 'dockerd reachable' }); } +/** The unit's cgroup directory, as the node sees it. */ +async function unitCgroup(container, unit) { + const r = await execInContainer(container, `systemctl show -p ControlGroup --value ${unit}`); + const group = r.stdout.trim(); + if (r.exitCode !== 0 || !group) throw new Error(`unit ${unit} has no cgroup: is it running?`); + return `/sys/fs/cgroup${group}`; +} + +/** + * Stop every process of a unit where it stands, alive, until thawUnit: a service + * that cannot act at all, as a wedged or memory-starved one cannot. Writes the + * cgroup's freeze switch directly, because `systemctl freeze` waits for the freeze + * to complete, and a thread held in the kernel (on a frozen filesystem) completes + * it only once it is released. Such a thread stays held, unchanged, and every + * other thread of the unit stops at once. The env's teardown thaws it + * (host-stall.thawEveryNode). + */ +export async function freezeUnit(container, unit) { + const freeze = `${await unitCgroup(container, unit)}/cgroup.freeze`; + const r = await execInContainer(container, `echo 1 > ${freeze} && cat ${freeze}`); + if (r.exitCode !== 0 || r.stdout.trim() !== '1') throw new Error(`freezeUnit ${unit}: ${(r.stderr || r.stdout).trim()}`); +} + +/** Let a unit frozen by freezeUnit run again. */ +export async function thawUnit(container, unit) { + const freeze = `${await unitCgroup(container, unit)}/cgroup.freeze`; + const r = await execInContainer(container, `echo 0 > ${freeze} && cat ${freeze}`); + if (r.exitCode !== 0 || r.stdout.trim() !== '0') throw new Error(`thawUnit ${unit}: ${(r.stderr || r.stdout).trim()}`); +} + /** * Read a unit's journal. `processOnly` restricts to the unit's own output * stream (`_TRANSPORT=stdout`, which carries both stdout and stderr) — without From 47aeb497351cbdf4bb6b9df05d868adb8991fe13 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 07:11:12 +0100 Subject: [PATCH 217/369] test(harness): 3302 and 3303 assert on the test event stream, and 3303 kills an app docker cannot stop Each case asserts that the dependency was held, read outside FluxOS, before what the node does about it. 3303's last case freezes the node's docker as well as its disk, so the app's removal cannot stop it and the node kills it through its slice. Co-Authored-By: Claude Opus 5.5 --- ...t-keeps-timing-out-takes-on-no-new-apps.js | 36 ++-- ...lled-host-takes-the-node-out-of-service.js | 195 +++++++++++------- 2 files changed, 145 insertions(+), 86 deletions(-) diff --git a/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js index dc14dbb5ac..e87504283c 100644 --- a/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js +++ b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js @@ -2,7 +2,7 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { getAppContainerStatus } from '../framework/container.js'; -import { waitFor, waitForUp, assertNoEvent } from '../framework/wait.js'; +import { waitForUp, assertNoEvent } from '../framework/wait.js'; import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; import { holdCommands, releaseCommands } from '../framework/mongo-control.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -17,11 +17,8 @@ const CALL_TIMEOUT_MS = 5 * 1000; const SUSPECT_WINDOW_MS = 60 * 1000; // Long enough for many round trips to run out on every node. const HOLD_MS = 45 * 1000; -// The driver's own words for a round trip and for a pooled-connection wait that ran -// out, as suite 3301 reads them: the canary that the calls failed, on any tree. -const DATABASE_GAVE_NO_ANSWER = /connection \d+ to \S+ timed out|Timed out while checking out a connection from connection pool/; -const HELD = /Placement hold set by databaseSuspect/; -const RELEASED = /Placement hold cleared by databaseSuspect/; +const OWNER = 'databaseSuspect'; +const byOwner = (data) => data.owner === OWNER; function step(what) { console.log(`# 3302 ${new Date().toISOString()} ${what}`); @@ -35,6 +32,15 @@ describe('a database that keeps timing out takes on no new apps', function () { let anchors; let held = false; let releasedAt; + let heldFrom = {}; + + // Each node that held itself back during the stall, with the event that says so. + function nodesThatHeld() { + return env.clients.map((client, i) => ({ + i, + set: client.getEventBuffer().find((e) => e.event === 'placementHold:set' && e.id > anchors[i] && byOwner(e.data)), + })).filter(({ set }) => set); + } before(async function () { this.timeout(420000 + HOLD_MS); @@ -56,7 +62,7 @@ describe('a database that keeps timing out takes on no new apps', function () { await new Promise((resolve) => { setTimeout(resolve, HOLD_MS); }); } finally { step('releasing the commands'); - await releaseCommands(); + heldFrom = await releaseCommands(); held = false; releasedAt = Date.now(); step('released'); @@ -69,13 +75,13 @@ describe('a database that keeps timing out takes on no new apps', function () { await env?.teardown(); }); - it('reached FluxOS: a node logged a call the database did not answer', function () { - const stalled = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); - expect(stalled, 'no node logged a database call that timed out, so nothing below was tested').to.not.be.empty; + it('reached FluxOS: the database held the nodes\' calls unanswered', function () { + const stalled = env.clients.filter((client) => heldFrom[client.ip] > 0); + expect(stalled, `the database held no node's call (held: ${JSON.stringify(heldFrom)}), so nothing below was tested`).to.not.be.empty; }); it('holds a node whose database kept timing out back from new placements', function () { - const holding = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, HELD)); + const holding = nodesThatHeld(); expect(holding, 'no node held itself back while its database left its calls unanswered').to.not.be.empty; }); @@ -90,11 +96,11 @@ describe('a database that keeps timing out takes on no new apps', function () { it('lifts the hold a window after the last round trip that ran out', async function () { this.timeout(SUSPECT_WINDOW_MS + 120000); - const holding = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, HELD)); + const holding = nodesThatHeld(); expect(holding, 'no node held itself back, so there is no hold to lift').to.not.be.empty; - await waitFor(() => holding.every((i) => env.nodeLogCount(i, RELEASED) >= env.nodeLogCount(i, HELD)), { - timeout: SUSPECT_WINDOW_MS + 90000, interval: 2000, label: 'every held node lifts its hold', - }); + await Promise.all(holding.map(async ({ i, set }) => { + await env.clients[i].waitForEvent('placementHold:cleared', byOwner, SUSPECT_WINDOW_MS + 90000, { afterId: set.id }); + })); expect(Date.now() - releasedAt, 'a hold lifted before its window could have passed').to.be.at.least(SUSPECT_WINDOW_MS - CALL_TIMEOUT_MS * 2); }); }); diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index cb95fddb65..c1986bd35d 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -2,12 +2,16 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { execInContainer, getAppContainerStatus } from '../framework/container.js'; -import { waitFor, waitForUp } from '../framework/wait.js'; -import { bootAndPeer, seedTestApp } from '../framework/reconciler-suite.js'; +import { waitForUp, waitForAppInstalled, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, seedTestApp, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { buildSeedableTestApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; import { - freezeDocker, thawDocker, stuckThreads, processesThatCanRun, + freezeDocker, thawDocker, thawEveryNode, stuckThreads, processesThatCanRun, } from '../framework/host-stall.js'; -import { journalCount, restartFluxos } from '../framework/systemd-control.js'; +import { + DOCKERD_UNIT, freezeUnit, restartFluxos, +} from '../framework/systemd-control.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // A host that stops returning the disk under docker, on a real kernel: the @@ -17,35 +21,40 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // // A thread held for one reading interval holds the node back from new placements, // and the hold lifts when the thread runs again. Held for stuckIntervals intervals, -// the node goes out of service until it reboots, and removes every app: the normal -// removal, which needs docker and so does not return, runs out of time, and the -// app's processes are killed through its slice without docker. +// the node goes out of service until it reboots, and removes every app within a +// deadline each. On a frozen disk docker still delivers the stop signal, so the +// app is stopped though its removal runs out. With docker frozen as well +// (systemd-control.freezeUnit), docker cannot act at all, and the app is killed +// through its slice without docker. // // The nodes run systemd, so docker uses its systemd cgroup driver and every app's // containers live in the app's own slice, as on a production node. +// +// Each case first asserts, independently of FluxOS, that docker was held: on a tree +// without the detector that canary passes and the assertions after it fail. const READ_INTERVAL_MS = 5 * 1000; const STUCK_INTERVALS = 6; const REMOVAL_DEADLINE_MS = 20 * 1000; const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; +const OWNER = 'hostStall'; -const HELD = 'Placement hold set by hostStall'; -const RELEASED = 'Placement hold cleared by hostStall'; -const OUT_OF_SERVICE = 'Sticky DOS set by hostStall'; +const byOwner = (data) => data.owner === OWNER; +const outOfServiceForTheStall = (data) => data.dosState >= 100 + && String(data.dosMessage).includes('Host stall') && String(data.dosMessage).includes('dockerd'); +const sliceOf = (appName) => `/sys/fs/cgroup/flux.slice/flux-${appName}.slice`; function step(what) { console.log(`# 3303 ${new Date().toISOString()} ${what}`); } -const fluxosLog = (container, text) => journalCount(container, 'fluxos', text, { processOnly: true }); - describe('a stalled host takes the node out of service', function () { let env; dumpLogsOnFailure(() => env); const appName = `e2ehoststall${Date.now()}`; let node; + let client; let container; - let frozen = false; before(async function () { this.timeout(480000); @@ -62,13 +71,13 @@ describe('a stalled host takes the node out of service', function () { }); await bootAndPeer(env); ({ index: node } = await seedTestApp(env, { name: appName, logEveryMs: 200 })); - ({ container } = env.clients[node]); - await waitForUp(env.clients[node], appName, 'the app is up'); + client = env.clients[node]; + ({ container } = client); + await waitForUp(client, appName, 'the app is up'); }); after(async function () { this.timeout(60000); - if (frozen) await thawDocker(container).catch(() => {}); await env?.teardown(); }); @@ -79,102 +88,146 @@ describe('a stalled host takes the node out of service', function () { }); describe('a stall shorter than the threshold', function () { - let counts; - let stuck; + let anchor; before(async function () { - this.timeout(STALL_MS * 2); - counts = { held: await fluxosLog(container, HELD), released: await fluxosLog(container, RELEASED), out: await fluxosLog(container, OUT_OF_SERVICE) }; + anchor = client.getLastEventId(); step('freezing'); await freezeDocker(container); - frozen = true; - try { - stuck = await stuckThreads(container, 'dockerd'); - await waitFor(async () => (await fluxosLog(container, HELD)) > counts.held, { - timeout: READ_INTERVAL_MS * 3, interval: 1000, label: 'the node holds itself back', - }); - } finally { - step('thawing'); - await thawDocker(container); - frozen = false; - } - }); - - it('held docker in the kernel', function () { + }); + + after(async function () { + await thawEveryNode([client]); + }); + + it('held docker in the kernel', async function () { + const stuck = await stuckThreads(container, 'dockerd'); expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; }); + it('holds the node back from new placements', async function () { + this.timeout(READ_INTERVAL_MS * 4); + await client.waitForEvent('placementHold:set', byOwner, READ_INTERVAL_MS * 3, { afterId: anchor }); + }); + it('lifts the hold once docker runs again, and stays in service', async function () { this.timeout(READ_INTERVAL_MS * 6); - await waitFor(async () => (await fluxosLog(container, RELEASED)) > counts.released, { - timeout: READ_INTERVAL_MS * 4, interval: 1000, label: 'the hold lifts', - }); - expect(await fluxosLog(container, OUT_OF_SERVICE), 'the node went out of service for a stall it recovered from').to.equal(counts.out); + step('thawing'); + await thawDocker(container); + await client.waitForEvent('placementHold:cleared', byOwner, READ_INTERVAL_MS * 4, { afterId: anchor }); + await assertNoEvent(client, 'dos:changed', (data) => data.dosState >= 100, 0, { afterId: anchor }); const app = await getAppContainerStatus(container, appName); expect(app && app.status.startsWith('Up'), 'the app is still up').to.equal(true); }); }); describe('a stall past the threshold', function () { - let stuck; + let anchor; before(async function () { - this.timeout(STALL_MS * 4); - const out = await fluxosLog(container, OUT_OF_SERVICE); + anchor = client.getLastEventId(); step('freezing'); await freezeDocker(container); - frozen = true; - stuck = await stuckThreads(container, 'dockerd'); - await waitFor(async () => (await fluxosLog(container, OUT_OF_SERVICE)) > out, { - timeout: STALL_MS + READ_INTERVAL_MS * 4, interval: 2000, label: 'the node goes out of service', - }); - step('out of service'); }); - it('held docker in the kernel', function () { + after(async function () { + await thawEveryNode([client]); + }); + + it('held docker in the kernel', async function () { + const stuck = await stuckThreads(container, 'dockerd'); expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; }); it('takes the node out of service, naming the stall', async function () { - const { data } = await env.clients[node].getDOSState(); - expect(data.dosState).to.be.at.least(100); - expect(data.dosMessage).to.include('Host stall').and.include('dockerd'); + this.timeout(STALL_MS + READ_INTERVAL_MS * 6); + await client.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }); + step('out of service'); }); it('leaves nothing of the app that can run, though its removal could not finish', async function () { this.timeout(REMOVAL_DEADLINE_MS * 4); - // the node's two words for an app with nothing left that can run - killed, or - // stopped before its removal ran out - and its one for an app it could not stop - await waitFor(async () => (await fluxosLog(container, `${appName} can no longer run`)) > 0 - || (await fluxosLog(container, `${appName} was not removed`)) > 0 - || (await fluxosLog(container, `${appName} could not be stopped`)) > 0, { - timeout: REMOVAL_DEADLINE_MS * 3, interval: 2000, label: 'the node says what is left of the app', - }); - const ranOut = (await fluxosLog(container, `${appName} was not removed`)) + (await fluxosLog(container, `${appName} still runs after its removal`)); - expect(ranOut, 'the app\'s removal finished, so its stall was never what removed it').to.be.above(0); - expect(await fluxosLog(container, `${appName} could not be stopped`), 'the node could not stop the app').to.equal(0); - expect(await processesThatCanRun(container, `/sys/fs/cgroup/flux.slice/flux-${appName}.slice`), 'a process of the app can still run').to.deep.equal([]); + const { data } = await client.waitForEvent('hostStall:appUnloaded', (d) => d.app === appName, REMOVAL_DEADLINE_MS * 3, { afterId: anchor }); + expect(data.removal, 'the app\'s removal finished, so its stall was never what removed it').to.not.equal('removed'); + expect(data.runnable, 'the node reports a process of the app that can still run').to.deep.equal([]); + expect(await processesThatCanRun(container, sliceOf(appName)), 'a process of the app can still run').to.deep.equal([]); }); it('stays out of service through a FluxOS restart in the same boot', async function () { this.timeout(300000); - const restored = await fluxosLog(container, OUT_OF_SERVICE); + const restartAnchor = client.getLastEventId(); step('restarting FluxOS'); await restartFluxos(container, { timeout: 120000, readyTimeoutMs: 180000 }); - await waitFor(async () => (await fluxosLog(container, OUT_OF_SERVICE)) > restored, { - timeout: 60000, interval: 2000, label: 'the restarted node restores its trip', - }); - const { data } = await env.clients[node].getDOSState(); - expect(data.dosState, 'a FluxOS restart returned the node to service').to.be.at.least(100); + await client.waitForEvent('dos:changed', outOfServiceForTheStall, 60000, { afterId: restartAnchor }); }); it('stays out of service once docker runs again', async function () { this.timeout(READ_INTERVAL_MS * 6); + const thawAnchor = client.getLastEventId(); + step('thawing'); await thawDocker(container); - frozen = false; - await new Promise((resolve) => { setTimeout(resolve, READ_INTERVAL_MS * 3); }); - const { data } = await env.clients[node].getDOSState(); + await assertNoEvent(client, 'dos:changed', (data) => data.dosState < 100, READ_INTERVAL_MS * 3, { afterId: thawAnchor }); + const { data } = await client.getDOSState(); expect(data.dosState, 'the node returned to service without a reboot').to.be.at.least(100); }); }); + + describe('a stall past the threshold, with docker unable to act at all', function () { + const killedName = `e2ehoststallkill${Date.now()}`; + let killClient; + let killContainer; + let anchor; + + before(async function () { + this.timeout(240000); + // on a node of its own: the node above is out of service until it reboots + const others = env.clients.map((_, i) => i).filter((i) => i !== node); + await pushTestApp(killedName); + const app = await buildSeedableTestApp({ name: killedName, logEveryMs: 200, instances: 1 }); + await seedSpawnerApp(env, app, others); + const index = await Promise.any(others.map(async (i) => { + await waitForAppInstalled(env.clients[i], killedName, 120000); + return i; + })); + killClient = env.clients[index]; + ({ container: killContainer } = killClient); + await waitForUp(killClient, killedName, 'the app is up'); + anchor = killClient.getLastEventId(); + step(`freezing node ${index}`); + await freezeDocker(killContainer); + }); + + after(async function () { + if (killClient) await thawEveryNode([killClient]); + }); + + it('held docker in the kernel', async function () { + const stuck = await stuckThreads(killContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + }); + + it('left docker unable to answer', async function () { + this.timeout(30000); + await freezeUnit(killContainer, DOCKERD_UNIT); + const answer = await execInContainer(killContainer, 'timeout 5 docker version'); + expect(answer.exitCode, 'docker still answers, so it could still stop the app itself').to.not.equal(0); + const stuck = await stuckThreads(killContainer, 'dockerd'); + expect(stuck, 'docker\'s held thread was released by its freeze, so the detector has nothing to read').to.not.be.empty; + }); + + it('takes the node out of service', async function () { + this.timeout(STALL_MS + READ_INTERVAL_MS * 6); + await killClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }); + }); + + it('kills the app its removal could not stop, and nothing of it can run', async function () { + // every app on the node is removed in turn, each within its deadline + this.timeout(REMOVAL_DEADLINE_MS * 7); + const { data } = await killClient.waitForEvent('hostStall:appUnloaded', (d) => d.app === killedName, REMOVAL_DEADLINE_MS * 6, { afterId: anchor }); + expect(data.removal, 'the app\'s removal finished, so docker could act').to.not.equal('removed'); + expect(data.killed, 'the node never killed the app').to.equal(true); + expect(data.runnable, 'the node reports a process of the app that can still run').to.deep.equal([]); + expect(await processesThatCanRun(killContainer, sliceOf(killedName)), 'a process of the app can still run').to.deep.equal([]); + }); + }); }); From 464f4936d5be8dc74fdd724c9268e1aa7077256e Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 07:30:56 +0100 Subject: [PATCH 218/369] test(harness): 3303's kill case asserts the app could still run while docker was frozen and when the node went out of service Co-Authored-By: Claude Opus 5.5 --- .../3303-a-stalled-host-takes-the-node-out-of-service.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index c1986bd35d..e276638058 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -177,6 +177,7 @@ describe('a stalled host takes the node out of service', function () { let killClient; let killContainer; let anchor; + let runnableAtTrip; before(async function () { this.timeout(240000); @@ -213,16 +214,20 @@ describe('a stalled host takes the node out of service', function () { expect(answer.exitCode, 'docker still answers, so it could still stop the app itself').to.not.equal(0); const stuck = await stuckThreads(killContainer, 'dockerd'); expect(stuck, 'docker\'s held thread was released by its freeze, so the detector has nothing to read').to.not.be.empty; + expect(await processesThatCanRun(killContainer, sliceOf(killedName)), 'the app is not running, so there is nothing to kill').to.not.be.empty; }); it('takes the node out of service', async function () { this.timeout(STALL_MS + READ_INTERVAL_MS * 6); await killClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }); + // read before the removal's deadline, while only a kill can end the app + runnableAtTrip = await processesThatCanRun(killContainer, sliceOf(killedName)); }); it('kills the app its removal could not stop, and nothing of it can run', async function () { // every app on the node is removed in turn, each within its deadline this.timeout(REMOVAL_DEADLINE_MS * 7); + expect(runnableAtTrip, 'nothing of the app could run when the node went out of service, so there was nothing to kill').to.not.be.empty; const { data } = await killClient.waitForEvent('hostStall:appUnloaded', (d) => d.app === killedName, REMOVAL_DEADLINE_MS * 6, { afterId: anchor }); expect(data.removal, 'the app\'s removal finished, so docker could act').to.not.equal('removed'); expect(data.killed, 'the node never killed the app').to.equal(true); From 6ec50bf45ba803efe5ea45c93a34c625ed98db8d Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 08:06:40 +0100 Subject: [PATCH 219/369] fix(apps): an app's cgroups are where systemd says they are The app's slice, flux.slice and each container's scope are found through systemd's ControlGroup for the unit, so the kill, its check and the boot's slice cleanup find them wherever systemd's tree starts: the filesystem's root on a host, and below it when systemd runs in a container. A container's scope inside the app's slice is read and killed with the slice. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 27 ++++++++++-------- ZelBack/src/services/appLifecycle/appSlice.js | 27 +++++++++++++----- tests/unit/appKill.test.js | 21 +++++++++++--- tests/unit/appSlice.test.js | 28 +++++++++++++------ 4 files changed, 73 insertions(+), 30 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index 49cbf2ebb5..b403b85ac5 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -3,10 +3,11 @@ * database, and say whether any of them can still run. * * An app's containers live in its slice (appSlice). A container created before - * the slice existed lives in its runtime's scope, `system.slice/docker-.scope`, - * and the container's name is read from the runtime's own record on disk, which - * stays readable while docker does not answer. Where those records are is read - * from docker at boot; until then it is docker's default for the node's kind. + * the slice existed lives in its runtime's scope, `docker-.scope`, and the + * container's name is read from the runtime's own record on disk, which stays + * readable while docker does not answer. Where those records are is read from + * docker at boot; until then it is docker's default for the node's kind. Where + * each cgroup is, systemd says (appSlice.controlGroup). * * A process held in the kernel is not removed by a kill: it keeps the kill * pending and dies the moment the kernel lets it go, without running the app's @@ -93,14 +94,14 @@ async function containersOnDisk() { } /** - * The scopes of the app's containers that live outside its slice. + * The scope units of the app's containers, whether in its slice or not. * @param {string} appName - * @returns {Promise} Cgroup directories. + * @returns {Promise} */ -async function legacyScopes(appName) { +async function scopesOf(appName) { return (await containersOnDisk()) .filter(({ name }) => belongsTo(name, appName)) - .map(({ id }) => path.join(paths.cgroupRoot, 'system.slice', `docker-${id}.scope`)); + .map(({ id }) => `docker-${id}.scope`); } /** @@ -110,8 +111,11 @@ async function legacyScopes(appName) { */ async function cgroupsOf(appName) { const unit = appSlice.sliceName(appName); - const candidates = [...(unit ? [appSlice.cgroupOf(unit)] : []), ...await legacyScopes(appName)]; - const present = await Promise.all(candidates.map((dir) => fs.access(path.join(dir, 'cgroup.procs')).then(() => dir, () => null))); + const units = [...(unit ? [unit] : []), ...await scopesOf(appName)]; + const found = (await Promise.all(units.map(appSlice.controlGroup))).filter(Boolean); + // a scope inside the app's slice is read and killed with the slice + const outermost = [...new Set(found)].filter((dir, _, all) => !all.some((other) => dir.startsWith(`${other}/`))); + const present = await Promise.all(outermost.map((dir) => fs.access(path.join(dir, 'cgroup.procs')).then(() => dir, () => null))); return present.filter(Boolean); } @@ -178,7 +182,8 @@ async function stillRunnable(appName) { */ async function appsOnDisk() { const names = new Set(); - const slices = await fs.readdir(path.join(paths.cgroupRoot, 'flux.slice')).catch(() => []); + const parent = await appSlice.controlGroup('flux.slice'); + const slices = parent ? await fs.readdir(parent).catch(() => []) : []; slices.filter((entry) => /^flux-.+\.slice$/.test(entry)) .forEach((entry) => names.add(entry.slice('flux-'.length, -'.slice'.length).replace(/\\x2d/g, '-'))); (await containersOnDisk()).map(({ name }) => appOf(name)).filter(Boolean).forEach((name) => names.add(name)); diff --git a/ZelBack/src/services/appLifecycle/appSlice.js b/ZelBack/src/services/appLifecycle/appSlice.js index c605588977..d2a5152a53 100644 --- a/ZelBack/src/services/appLifecycle/appSlice.js +++ b/ZelBack/src/services/appLifecycle/appSlice.js @@ -7,6 +7,10 @@ * * A slice outlives its last container until it is stopped. Removing an app * stops its slice, and boot stops every slice left empty. + * + * Where a unit's cgroup is, systemd says: its tree starts wherever systemd itself + * runs, which is the filesystem's root on a host and below it when systemd runs + * in a container. */ const fs = require('fs').promises; @@ -16,6 +20,8 @@ const serviceHelper = require('../serviceHelper'); const PARENT = 'flux.slice'; const APP_NAME = /^[A-Za-z0-9-]+$/; +// systemd answers from memory; a question it has not answered by then gets no answer +const SYSTEMD_ANSWER_MS = 10 * 1000; let cgroupRoot = '/sys/fs/cgroup'; @@ -39,11 +45,16 @@ function sliceName(appName) { } /** - * @param {string} unit A slice under flux.slice. - * @returns {string} Its cgroup directory. + * The unit's cgroup directory, as systemd reports it. + * @param {string} unit Any systemd unit: a slice, a scope, a service. + * @returns {Promise} Null when systemd has no such unit, or no answer. */ -function cgroupOf(unit) { - return path.join(cgroupRoot, PARENT, unit); +async function controlGroup(unit) { + const { stdout, error } = await serviceHelper.runCommand('systemctl', { + params: ['show', '-p', 'ControlGroup', '--value', unit], timeout: SYSTEMD_ANSWER_MS, logError: false, + }); + const group = (stdout ?? '').trim(); + return error || !group ? null : path.join(cgroupRoot, group); } /** @@ -52,7 +63,8 @@ function cgroupOf(unit) { * null when the slice does not exist. */ async function isPopulated(unit) { - const events = await fs.readFile(path.join(cgroupOf(unit), 'cgroup.events'), 'utf8').catch(() => null); + const dir = await controlGroup(unit); + const events = dir ? await fs.readFile(path.join(dir, 'cgroup.events'), 'utf8').catch(() => null) : null; if (events === null) return null; return /^populated 1$/m.test(events); } @@ -87,7 +99,8 @@ async function release(appName) { * @returns {Promise} How many were stopped. */ async function stopEmptySlices() { - const entries = await fs.readdir(path.join(cgroupRoot, PARENT)).catch(() => []); + const parent = await controlGroup(PARENT); + const entries = parent ? await fs.readdir(parent).catch(() => []) : []; const slices = entries.filter((entry) => entry.startsWith('flux-') && entry.endsWith('.slice')); let stopped = 0; // eslint-disable-next-line no-restricted-syntax @@ -101,7 +114,7 @@ async function stopEmptySlices() { module.exports = { sliceName, - cgroupOf, + controlGroup, release, stopEmptySlices, useCgroupRoot, diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index 8fe8bcce23..1b6e57b7a0 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -12,19 +12,28 @@ describe('appKill tests', () => { let procRoot; let dockerRoot; let killed; + // unit -> its cgroup, as systemd reports it + let units; const id = (n) => String(n).repeat(64).slice(0, 64); + // systemd runs in a container here, so its tree starts below the filesystem's root + const SYSTEMD_ROOT = '/init'; function cgroup(rel, pids) { - const dir = path.join(cgroupRoot, rel); + const dir = path.join(cgroupRoot, SYSTEMD_ROOT, rel); fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, 'cgroup.procs'), pids.map((pid) => `${pid}\n`).join('')); fs.writeFileSync(path.join(dir, 'cgroup.kill'), ''); + units.set(path.basename(rel), path.join(SYSTEMD_ROOT, rel)); + if (rel.startsWith('flux.slice/')) units.set('flux.slice', path.join(SYSTEMD_ROOT, 'flux.slice')); return dir; } - function container(n, name, pids) { + function record(n, name, pid = 0) { fs.mkdirSync(path.join(dockerRoot, 'containers', id(n)), { recursive: true }); - fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pids[0] ?? 0) } })); + fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pid) } })); + } + function container(n, name, pids) { + record(n, name, pids[0]); return cgroup(path.join('system.slice', `docker-${id(n)}.scope`), pids); } // An app's slice, as the kernel lays it out: no process of its own, and each @@ -39,8 +48,10 @@ describe('appKill tests', () => { fs.writeFileSync(path.join(procRoot, pid, 'status'), `Name:\tapp\nSigPnd:\t${sigPnd}\nShdPnd:\t${shdPnd}\n`); } - // what sudo would do: read and list as root, and write to cgroup.kill + // what sudo would do: read and list as root, and write to cgroup.kill; and + // what systemd answers about a unit's cgroup function asRoot(cmd, { params }) { + if (cmd === 'systemctl') return Promise.resolve({ stdout: `${units.get(params[4]) ?? ''}\n`, error: null }); if (cmd === 'ls') return Promise.resolve({ stdout: fs.readdirSync(params[1]).join('\n'), error: null }); if (cmd === 'cat') { try { @@ -64,6 +75,7 @@ describe('appKill tests', () => { fs.mkdirSync(path.join(dockerRoot, 'containers'), { recursive: true }); appKill.usePaths({ cgroupRoot, procRoot, dockerRoot }); killed = []; + units = new Map(); sinon.stub(serviceHelper, 'runCommand').callsFake(asRoot); }); @@ -76,6 +88,7 @@ describe('appKill tests', () => { describe('kill', () => { it('kills the app\'s slice and the scopes of its containers made before slices, and no other app\'s', async () => { const appSliceDir = slice('flux-my\\x2dapp.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + record(9, 'fluxdb_my-app', '10'); const web = container(1, 'fluxweb_my-app', ['11']); const single = container(2, 'fluxmy-app', ['12']); container(3, 'fluxweb_my-app2', ['13']); diff --git a/tests/unit/appSlice.test.js b/tests/unit/appSlice.test.js index 510437b878..5dba6961ed 100644 --- a/tests/unit/appSlice.test.js +++ b/tests/unit/appSlice.test.js @@ -9,18 +9,30 @@ const appSlice = require('../../ZelBack/src/services/appLifecycle/appSlice'); describe('appSlice tests', () => { let root; let runCommand; + // unit -> its cgroup, as systemd reports it + let units; + // systemd runs in a container here, so its tree starts below the filesystem's root + const SYSTEMD_ROOT = '/init'; function slice(unit, populated) { - const dir = path.join(root, 'flux.slice', unit); + const dir = path.join(root, SYSTEMD_ROOT, 'flux.slice', unit); fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, 'cgroup.events'), `populated ${populated ? 1 : 0}\nfrozen 0\n`); + units.set('flux.slice', path.join(SYSTEMD_ROOT, 'flux.slice')); + units.set(unit, path.join(SYSTEMD_ROOT, 'flux.slice', unit)); } - const stopped = () => runCommand.getCalls().map((call) => call.args[1].params[1]); + const stops = () => runCommand.getCalls().filter((call) => call.args[1].params[0] === 'stop'); + const stopped = () => stops().map((call) => call.args[1].params[1]); + let stopResult; beforeEach(() => { root = fs.mkdtempSync(path.join(os.tmpdir(), 'appslice-')); appSlice.useCgroupRoot(root); - runCommand = sinon.stub(serviceHelper, 'runCommand').resolves({ error: null }); + units = new Map(); + stopResult = { error: null }; + runCommand = sinon.stub(serviceHelper, 'runCommand').callsFake((cmd, { params }) => Promise.resolve(params[0] === 'show' + ? { stdout: `${units.get(params[4]) ?? ''}\n`, error: null } + : stopResult)); }); afterEach(() => { @@ -52,7 +64,7 @@ describe('appSlice tests', () => { await appSlice.release('my-app'); - sinon.assert.calledOnceWithExactly(runCommand, 'systemctl', { runAsRoot: true, params: ['stop', 'flux-my\\x2dapp.slice'] }); + expect(stops().map((call) => call.args)).to.deep.equal([['systemctl', { runAsRoot: true, params: ['stop', 'flux-my\\x2dapp.slice'] }]]); }); it('leaves a slice a process still lives in', async () => { @@ -60,13 +72,13 @@ describe('appSlice tests', () => { await appSlice.release('busy'); - sinon.assert.notCalled(runCommand); + expect(stops()).to.be.empty; }); it('does nothing for an app with no slice', async () => { await appSlice.release('absent'); - sinon.assert.notCalled(runCommand); + expect(stops()).to.be.empty; }); }); @@ -85,14 +97,14 @@ describe('appSlice tests', () => { it('counts no slice systemd could not stop', async () => { slice('flux-gone.slice', false); - runCommand.resolves({ error: new Error('unit busy') }); + stopResult = { error: new Error('unit busy') }; expect(await appSlice.stopEmptySlices()).to.equal(0); }); it('does nothing when there is no flux.slice', async () => { expect(await appSlice.stopEmptySlices()).to.equal(0); - sinon.assert.notCalled(runCommand); + expect(stops()).to.be.empty; }); }); }); From 9ea436f12b1b3290c0ec5e744dcc4f8d6136ba79 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 08:07:14 +0100 Subject: [PATCH 220/369] test(harness): 3303 reads an app's processes from where the node's systemd says its slice is processesThatCanRun takes a unit and asks the node's systemd for its cgroup, and a unit systemd has no cgroup for is an error. 3303's first case compares the container's cgroup against the slice's exact path. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/host-stall.js | 15 +++++++++------ test-infra/runner/framework/systemd-control.js | 4 ++-- ...-stalled-host-takes-the-node-out-of-service.js | 10 ++++++---- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index 2c94af09f4..b37f9ccd0a 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -7,6 +7,7 @@ // mounts its docker data root on a filesystem of its own. The env's teardown thaws // it, so a node a failing test left frozen can still be stopped. import { execInContainer } from './container.js'; +import { unitCgroup } from './systemd-control.js'; export const DOCKER_ROOT = '/mnt/appdata/docker'; @@ -44,15 +45,17 @@ export async function thawEveryNode(clients) { } /** - * The processes in a cgroup and every cgroup below it that could still run: - * neither gone nor with SIGKILL pending. Read inside the node, independently of - * FluxOS. + * The processes in a unit's cgroup and every cgroup below it that could still + * run: neither gone nor with SIGKILL pending. Read inside the node, independently + * of FluxOS, from where the node's systemd says the unit is. Throws when systemd + * has no cgroup for the unit, which is never an answer of "nothing can run". * @param {object} container - * @param {string} cgroupDir Under /sys/fs/cgroup. + * @param {string} unit A systemd unit, e.g. an app's slice. * @returns {Promise} Their pids; empty when nothing in it can run again. */ -export async function processesThatCanRun(container, cgroupDir) { - const script = `[ -d '${cgroupDir}' ] || exit 0; for p in $(find '${cgroupDir}' -name cgroup.procs -exec cat {} +); do ` +export async function processesThatCanRun(container, unit) { + const cgroupDir = await unitCgroup(container, unit); + const script = `for p in $(find '${cgroupDir}' -name cgroup.procs -exec cat {} +); do ` + 's=/proc/$p/status; [ -r $s ] || continue; ' + 'k=$(awk \'/^(SigPnd|ShdPnd):/{print $2}\' $s | while read m; do [ $(( 0x$m & 0x100 )) -ne 0 ] && echo y; done); ' + '[ -z "$k" ] && echo $p; done; true'; diff --git a/test-infra/runner/framework/systemd-control.js b/test-infra/runner/framework/systemd-control.js index 1baae73288..60bfab9e6f 100644 --- a/test-infra/runner/framework/systemd-control.js +++ b/test-infra/runner/framework/systemd-control.js @@ -105,8 +105,8 @@ export async function resumeDockerd(container, opts = {}) { { timeout: opts.timeout ?? 60000, interval: 500, label: 'dockerd reachable' }); } -/** The unit's cgroup directory, as the node sees it. */ -async function unitCgroup(container, unit) { +/** The unit's cgroup directory, as the node's systemd reports it. Throws when it has none. */ +export async function unitCgroup(container, unit) { const r = await execInContainer(container, `systemctl show -p ControlGroup --value ${unit}`); const group = r.stdout.trim(); if (r.exitCode !== 0 || !group) throw new Error(`unit ${unit} has no cgroup: is it running?`); diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index e276638058..f2b27a2a35 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -10,7 +10,7 @@ import { freezeDocker, thawDocker, thawEveryNode, stuckThreads, processesThatCanRun, } from '../framework/host-stall.js'; import { - DOCKERD_UNIT, freezeUnit, restartFluxos, + DOCKERD_UNIT, freezeUnit, restartFluxos, unitCgroup, } from '../framework/systemd-control.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -42,7 +42,8 @@ const OWNER = 'hostStall'; const byOwner = (data) => data.owner === OWNER; const outOfServiceForTheStall = (data) => data.dosState >= 100 && String(data.dosMessage).includes('Host stall') && String(data.dosMessage).includes('dockerd'); -const sliceOf = (appName) => `/sys/fs/cgroup/flux.slice/flux-${appName}.slice`; +// the app names here have no '-', which a slice name escapes +const sliceOf = (appName) => `flux-${appName}.slice`; function step(what) { console.log(`# 3303 ${new Date().toISOString()} ${what}`); @@ -83,8 +84,9 @@ describe('a stalled host takes the node out of service', function () { it('runs the app in its own slice', async function () { const pid = (await execInContainer(container, `docker inspect -f '{{.State.Pid}}' flux${appName}_${appName}`)).stdout.trim(); - const cgroup = (await execInContainer(container, `cat /proc/${pid}/cgroup`)).stdout; - expect(cgroup, 'the app container is not in its slice, so the kill below has nothing to find').to.include(`/flux.slice/flux-${appName}.slice/`); + const cgroup = (await execInContainer(container, `cat /proc/${pid}/cgroup`)).stdout.trim().replace(/^0::/, '/sys/fs/cgroup'); + const slice = await unitCgroup(container, sliceOf(appName)).catch(() => null); + expect(slice !== null && cgroup.startsWith(`${slice}/`), `the app container is in ${cgroup}, not in its slice ${slice}, so the kill below has nothing to find`).to.equal(true); }); describe('a stall shorter than the threshold', function () { From 1af6bc9de108b737d9d78d798593301946abd003 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 08:19:32 +0100 Subject: [PATCH 221/369] test(harness): 3303's canaries read the app's container where any tree puts it, and thawing a thawed docker is a no-op containerCgroup records the cgroup a container's main process lives in while docker answers; processesThatCanRun reads a cgroup directory and refuses one that does not exist. The kill case's canaries read the app's container, which exists with or without slices; the end state is read from the app's slice. thawDocker leaves a data root that is not frozen as it is. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/host-stall.js | 33 +++++++++++++------ ...lled-host-takes-the-node-out-of-service.js | 21 ++++++++---- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index b37f9ccd0a..d1caedf8a1 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -7,7 +7,6 @@ // mounts its docker data root on a filesystem of its own. The env's teardown thaws // it, so a node a failing test left frozen can still be stopped. import { execInContainer } from './container.js'; -import { unitCgroup } from './systemd-control.js'; export const DOCKER_ROOT = '/mnt/appdata/docker'; @@ -23,9 +22,10 @@ export async function freezeDocker(container) { await run(container, `fsfreeze -f ${DOCKER_ROOT}`, 'freezeDocker'); } -/** Let the node's docker write again. */ +/** Let the node's docker write again. A data root that is not frozen is left as it is. */ export async function thawDocker(container) { - await run(container, `fsfreeze -u ${DOCKER_ROOT}`, 'thawDocker'); + // fsfreeze refuses to thaw a filesystem that is not frozen, with EINVAL + await run(container, `out=$(fsfreeze -u ${DOCKER_ROOT} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }`, 'thawDocker'); } /** @@ -45,16 +45,29 @@ export async function thawEveryNode(clients) { } /** - * The processes in a unit's cgroup and every cgroup below it that could still - * run: neither gone nor with SIGKILL pending. Read inside the node, independently - * of FluxOS, from where the node's systemd says the unit is. Throws when systemd - * has no cgroup for the unit, which is never an answer of "nothing can run". + * The cgroup directory a container's main process lives in, read while docker + * still answers. + * @param {object} container The node. + * @param {string} name The container's name inside the node. + * @returns {Promise} Under /sys/fs/cgroup. + */ +export async function containerCgroup(container, name) { + const pid = (await run(container, `docker inspect -f '{{.State.Pid}}' ${name}`, `finding ${name}'s process`)).trim(); + const line = (await run(container, `cat /proc/${pid}/cgroup`, `reading ${name}'s cgroup`)).trim(); + return line.replace(/^0::/, '/sys/fs/cgroup'); +} + +/** + * The processes in a cgroup and every cgroup below it that could still run: + * neither gone nor with SIGKILL pending. Read inside the node, independently of + * FluxOS. Throws when the cgroup does not exist, which is never an answer of + * "nothing can run". * @param {object} container - * @param {string} unit A systemd unit, e.g. an app's slice. + * @param {string} cgroupDir Under /sys/fs/cgroup: systemd-control.unitCgroup, or containerCgroup. * @returns {Promise} Their pids; empty when nothing in it can run again. */ -export async function processesThatCanRun(container, unit) { - const cgroupDir = await unitCgroup(container, unit); +export async function processesThatCanRun(container, cgroupDir) { + await run(container, `test -d '${cgroupDir}'`, `${cgroupDir} does not exist`); const script = `for p in $(find '${cgroupDir}' -name cgroup.procs -exec cat {} +); do ` + 's=/proc/$p/status; [ -r $s ] || continue; ' + 'k=$(awk \'/^(SigPnd|ShdPnd):/{print $2}\' $s | while read m; do [ $(( 0x$m & 0x100 )) -ne 0 ] && echo y; done); ' diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index f2b27a2a35..e2cb1418da 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -7,7 +7,7 @@ import { bootAndPeer, seedTestApp, seedSpawnerApp } from '../framework/reconcile import { buildSeedableTestApp } from '../framework/seed-helper.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { - freezeDocker, thawDocker, thawEveryNode, stuckThreads, processesThatCanRun, + freezeDocker, thawDocker, thawEveryNode, stuckThreads, processesThatCanRun, containerCgroup, } from '../framework/host-stall.js'; import { DOCKERD_UNIT, freezeUnit, restartFluxos, unitCgroup, @@ -44,6 +44,11 @@ const outOfServiceForTheStall = (data) => data.dosState >= 100 && String(data.dosMessage).includes('Host stall') && String(data.dosMessage).includes('dockerd'); // the app names here have no '-', which a slice name escapes const sliceOf = (appName) => `flux-${appName}.slice`; +const containerOf = (appName) => `flux${appName}_${appName}`; +// what of the app can still run, read from its slice, which outlives its containers +async function sliceProcessesThatCanRun(container, appName) { + return processesThatCanRun(container, await unitCgroup(container, sliceOf(appName))); +} function step(what) { console.log(`# 3303 ${new Date().toISOString()} ${what}`); @@ -83,8 +88,7 @@ describe('a stalled host takes the node out of service', function () { }); it('runs the app in its own slice', async function () { - const pid = (await execInContainer(container, `docker inspect -f '{{.State.Pid}}' flux${appName}_${appName}`)).stdout.trim(); - const cgroup = (await execInContainer(container, `cat /proc/${pid}/cgroup`)).stdout.trim().replace(/^0::/, '/sys/fs/cgroup'); + const cgroup = await containerCgroup(container, containerOf(appName)); const slice = await unitCgroup(container, sliceOf(appName)).catch(() => null); expect(slice !== null && cgroup.startsWith(`${slice}/`), `the app container is in ${cgroup}, not in its slice ${slice}, so the kill below has nothing to find`).to.equal(true); }); @@ -152,7 +156,7 @@ describe('a stalled host takes the node out of service', function () { const { data } = await client.waitForEvent('hostStall:appUnloaded', (d) => d.app === appName, REMOVAL_DEADLINE_MS * 3, { afterId: anchor }); expect(data.removal, 'the app\'s removal finished, so its stall was never what removed it').to.not.equal('removed'); expect(data.runnable, 'the node reports a process of the app that can still run').to.deep.equal([]); - expect(await processesThatCanRun(container, sliceOf(appName)), 'a process of the app can still run').to.deep.equal([]); + expect(await sliceProcessesThatCanRun(container, appName), 'a process of the app can still run').to.deep.equal([]); }); it('stays out of service through a FluxOS restart in the same boot', async function () { @@ -180,6 +184,8 @@ describe('a stalled host takes the node out of service', function () { let killContainer; let anchor; let runnableAtTrip; + // the app's container's own cgroup, wherever this tree puts it + let appCgroup; before(async function () { this.timeout(240000); @@ -195,6 +201,7 @@ describe('a stalled host takes the node out of service', function () { killClient = env.clients[index]; ({ container: killContainer } = killClient); await waitForUp(killClient, killedName, 'the app is up'); + appCgroup = await containerCgroup(killContainer, containerOf(killedName)); anchor = killClient.getLastEventId(); step(`freezing node ${index}`); await freezeDocker(killContainer); @@ -216,14 +223,14 @@ describe('a stalled host takes the node out of service', function () { expect(answer.exitCode, 'docker still answers, so it could still stop the app itself').to.not.equal(0); const stuck = await stuckThreads(killContainer, 'dockerd'); expect(stuck, 'docker\'s held thread was released by its freeze, so the detector has nothing to read').to.not.be.empty; - expect(await processesThatCanRun(killContainer, sliceOf(killedName)), 'the app is not running, so there is nothing to kill').to.not.be.empty; + expect(await processesThatCanRun(killContainer, appCgroup), 'the app is not running, so there is nothing to kill').to.not.be.empty; }); it('takes the node out of service', async function () { this.timeout(STALL_MS + READ_INTERVAL_MS * 6); await killClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }); // read before the removal's deadline, while only a kill can end the app - runnableAtTrip = await processesThatCanRun(killContainer, sliceOf(killedName)); + runnableAtTrip = await processesThatCanRun(killContainer, appCgroup); }); it('kills the app its removal could not stop, and nothing of it can run', async function () { @@ -234,7 +241,7 @@ describe('a stalled host takes the node out of service', function () { expect(data.removal, 'the app\'s removal finished, so docker could act').to.not.equal('removed'); expect(data.killed, 'the node never killed the app').to.equal(true); expect(data.runnable, 'the node reports a process of the app that can still run').to.deep.equal([]); - expect(await processesThatCanRun(killContainer, sliceOf(killedName)), 'a process of the app can still run').to.deep.equal([]); + expect(await sliceProcessesThatCanRun(killContainer, killedName), 'a process of the app can still run').to.deep.equal([]); }); }); }); From c94893c85189072020876ae3d05a9a3458fbb66a Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 08:46:38 +0100 Subject: [PATCH 222/369] test(harness): a systemd-mode node can run FluxOS as an account that is not root createTestEnv({ systemdMode: true, unprivilegedNodes: [i] }) gives fluxos.service a drop-in naming the account, which also sets its HOME, and hands it the apps folder, as the default mode does. Co-Authored-By: Claude Opus 5.5 --- test-infra/entrypoint.sh | 15 +++++++++------ test-infra/runner/framework/test-env.js | 6 ------ test-infra/systemd/fluxos.service | 3 ++- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 72b0927994..b532a90392 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -253,12 +253,6 @@ fi # (/tmp/fluxos.pid — restartFluxos) does not exist under systemd, and # framework/systemd-control.js holds the equivalents. if [ "$FLUX_SYSTEMD_MODE" = "true" ]; then - # fluxos.service runs FluxOS as root. An unprivileged node is a different - # install shape, and booting it as root would test the wrong one. - if [ -n "$FLUX_FLUXOS_USER" ]; then - echo "[entrypoint] FATAL: systemd mode runs FluxOS as root; FLUX_FLUXOS_USER=$FLUX_FLUXOS_USER is not supported here" >&2 - exit 1 - fi # The Ubuntu base image's policy-rc.d refuses every service start a package # install asks for. A host has none, so a package installed here starts its @@ -282,6 +276,15 @@ if [ "$FLUX_SYSTEMD_MODE" = "true" ]; then mkdir -p /etc/systemd/system/multi-user.target.wants ln -sf /etc/systemd/system/dockerd.service /etc/systemd/system/multi-user.target.wants/dockerd.service ln -sf /etc/systemd/system/fluxos.service /etc/systemd/system/multi-user.target.wants/fluxos.service + # An unprivileged node runs FluxOS as the operator's account, as an operator's own + # install does; systemd gives the service that account's HOME, from which a legacy + # node derives its flux directory. The apps folder is the account's, and nothing + # else changes hands, as in the default mode below. + if [ -n "$FLUX_FLUXOS_USER" ]; then + mkdir -p /etc/systemd/system/fluxos.service.d + printf '[Service]\nUser=%s\n' "$FLUX_FLUXOS_USER" > /etc/systemd/system/fluxos.service.d/user.conf + chown "$(id -u "$FLUX_FLUXOS_USER"):$(id -g "$FLUX_FLUXOS_USER")" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" + fi if [ -n "$FLUX_SYNCTHING_HOST" ]; then ln -sf /etc/systemd/system/syncthing-forward.service /etc/systemd/system/multi-user.target.wants/syncthing-forward.service fi diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 3d1c9d7d86..2b0cd0e5a3 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -1245,12 +1245,6 @@ export async function createTestEnv({ if (aptBadSource && legacyNodes.length === 0) { throw new Error('createTestEnv: aptBadSource needs legacyNodes - no other node type runs apt'); } - // fluxos.service runs FluxOS as root, so an unprivileged node cannot be one. - // Refused here rather than at the node's own boot, where it would read as a - // node that never came up. - if (systemdMode && unprivilegedNodes.length) { - throw new Error('createTestEnv: systemdMode runs FluxOS as root - unprivilegedNodes is not supported with it'); - } // The boot-lock queue wait must not count against the suite's hook budget. // Mocha enforces a hook's timeout twice: the watchdog timer (which would fire // MID-QUEUE whenever the queue alone outlasts the budget), and a completion-time diff --git a/test-infra/systemd/fluxos.service b/test-infra/systemd/fluxos.service index 36a5fb37a2..1f2907f71f 100644 --- a/test-infra/systemd/fluxos.service +++ b/test-infra/systemd/fluxos.service @@ -6,7 +6,8 @@ Wants=dockerd.service [Service] # User= is what makes systemd set HOME, and a legacy node finds its install at -# $HOME/zelflux (the image links /root/zelflux to /flux). +# $HOME/zelflux (the image links /root/zelflux and /home/fluxuser/zelflux to +# /flux). An unprivileged node replaces it with a drop-in (entrypoint.sh). User=root WorkingDirectory=/flux # Container env does not cross into systemd services (the manager From b8ff30b26339cb6b5316fce853eabe5c12a69111 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 08:46:39 +0100 Subject: [PATCH 223/369] test(harness): 3303's kill case runs on a legacy node whose FluxOS is not root Co-Authored-By: Claude Opus 5.5 --- ...lled-host-takes-the-node-out-of-service.js | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index e2cb1418da..10a3831946 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -3,7 +3,7 @@ import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { execInContainer, getAppContainerStatus } from '../framework/container.js'; import { waitForUp, waitForAppInstalled, assertNoEvent } from '../framework/wait.js'; -import { bootAndPeer, seedTestApp, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { buildSeedableTestApp } from '../framework/seed-helper.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { @@ -28,7 +28,9 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // through its slice without docker. // // The nodes run systemd, so docker uses its systemd cgroup driver and every app's -// containers live in the app's own slice, as on a production node. +// containers live in the app's own slice, as on a production node. The node the +// kill case runs on is a legacy node whose FluxOS runs as the operator's account, +// not root: it reads docker's threads, asks systemd and kills through sudo. // // Each case first asserts, independently of FluxOS, that docker was held: on a tree // without the detector that canary passes and the assertions after it fail. @@ -38,6 +40,8 @@ const STUCK_INTERVALS = 6; const REMOVAL_DEADLINE_MS = 20 * 1000; const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; const OWNER = 'hostStall'; +const ARCANE = [0, 1, 2]; +const LEGACY_UNPRIVILEGED = 3; const byOwner = (data) => data.owner === OWNER; const outOfServiceForTheStall = (data) => data.dosState >= 100 @@ -54,6 +58,17 @@ function step(what) { console.log(`# 3303 ${new Date().toISOString()} ${what}`); } +// An app that writes a log line every 200 ms, installed on one of `indices` only. +async function installOn(env, name, indices) { + await pushTestApp(name); + const app = await buildSeedableTestApp({ name, logEveryMs: 200, instances: 1 }); + await seedSpawnerApp(env, app, indices); + return Promise.any(indices.map(async (i) => { + await waitForAppInstalled(env.clients[i], name, 120000); + return i; + })); +} + describe('a stalled host takes the node out of service', function () { let env; dumpLogsOnFailure(() => env); @@ -70,13 +85,15 @@ describe('a stalled host takes the node out of service', function () { tickerAutostart: false, systemdMode: true, freezableDockerRoot: [0, 1, 2, 3], + legacyNodes: [LEGACY_UNPRIVILEGED], + unprivilegedNodes: [LEGACY_UNPRIVILEGED], configOverrides: { hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS }, fluxapps: { appRemovalDeadlineMs: REMOVAL_DEADLINE_MS }, }, }); await bootAndPeer(env); - ({ index: node } = await seedTestApp(env, { name: appName, logEveryMs: 200 })); + node = await installOn(env, appName, ARCANE); client = env.clients[node]; ({ container } = client); await waitForUp(client, appName, 'the app is up'); @@ -178,7 +195,7 @@ describe('a stalled host takes the node out of service', function () { }); }); - describe('a stall past the threshold, with docker unable to act at all', function () { + describe('a stall past the threshold, with docker unable to act at all, on a legacy node whose FluxOS is not root', function () { const killedName = `e2ehoststallkill${Date.now()}`; let killClient; let killContainer; @@ -189,15 +206,7 @@ describe('a stalled host takes the node out of service', function () { before(async function () { this.timeout(240000); - // on a node of its own: the node above is out of service until it reboots - const others = env.clients.map((_, i) => i).filter((i) => i !== node); - await pushTestApp(killedName); - const app = await buildSeedableTestApp({ name: killedName, logEveryMs: 200, instances: 1 }); - await seedSpawnerApp(env, app, others); - const index = await Promise.any(others.map(async (i) => { - await waitForAppInstalled(env.clients[i], killedName, 120000); - return i; - })); + const index = await installOn(env, killedName, [LEGACY_UNPRIVILEGED]); killClient = env.clients[index]; ({ container: killContainer } = killClient); await waitForUp(killClient, killedName, 'the app is up'); @@ -211,6 +220,11 @@ describe('a stalled host takes the node out of service', function () { if (killClient) await thawEveryNode([killClient]); }); + it('runs FluxOS as an account that is not root', async function () { + const owner = await execInContainer(killContainer, 'stat -c %U /proc/$(systemctl show -p MainPID --value fluxos)'); + expect(owner.stdout.trim(), 'FluxOS runs as root, so nothing below tested an operator\'s own install').to.equal('fluxuser'); + }); + it('held docker in the kernel', async function () { const stuck = await stuckThreads(killContainer, 'dockerd'); expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; From cda53ab96d96f024a8ce3731e6080a6fe62550c2 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 09:01:02 +0100 Subject: [PATCH 224/369] fix(apps): the stall detector reads a dependency's namespace from a file any account may read Whether a process is in FluxOS's own PID namespace is read from the NSpid line of its status: a process as many namespaces deep as FluxOS is in its namespace. A FluxOS that is not root, as on an operator's own install, reads every dependency of root's the same as one that is. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 24 ++++++++++++++++++++---- tests/unit/hostStallService.test.js | 17 ++++++++++------- 2 files changed, 30 insertions(+), 11 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 35ae5ac953..274df14c94 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -18,7 +18,8 @@ * * WHAT IS READ. The threads of the processes FluxOS depends on, in FluxOS's own * PID namespace. An app container that runs one of the same programs has its - * own namespace and is not read. + * own namespace and is not read. Every file read is one any account may read of + * another's process, so a FluxOS that is not root reads the same. */ const fs = require('fs').promises; @@ -82,12 +83,27 @@ async function readThread(taskDir) { }; } +/** + * How many PID namespaces deep a process is, counted from the one this /proc + * shows: its status names its id in each, outermost first. A process in a + * namespace below FluxOS's has more than FluxOS; one beside it is not in this + * /proc at all. A kernel older than 4.1 writes no NSpid, and every process on it + * reads as one deep. + * @param {string|null} status The process's status file. + * @returns {number|null} Null when the status is gone. + */ +function namespaceDepth(status) { + if (status === null) return null; + const line = status.split('\n').find((entry) => entry.startsWith('NSpid:')); + return line ? line.trim().split(/\s+/).length - 1 : 1; +} + /** * Every thread of every dependency process in this PID namespace. * @returns {Promise>} */ async function readDependencyThreads() { - const ownNamespace = await fs.readlink(path.join(paths.procRoot, 'self', 'ns', 'pid')).catch(() => null); + const ownDepth = namespaceDepth(await readOrNull(path.join(paths.procRoot, 'self', 'status'))); const pids = (await fs.readdir(paths.procRoot)).filter((entry) => /^\d+$/.test(entry)); const threads = []; // eslint-disable-next-line no-restricted-syntax @@ -98,9 +114,9 @@ async function readDependencyThreads() { // eslint-disable-next-line no-continue if (!DEPENDENCIES.has(comm)) continue; // eslint-disable-next-line no-await-in-loop - const namespace = await fs.readlink(path.join(pidDir, 'ns', 'pid')).catch(() => null); + const depth = namespaceDepth(await readOrNull(path.join(pidDir, 'status'))); // eslint-disable-next-line no-continue - if (namespace !== ownNamespace) continue; + if (depth === null || depth !== ownDepth) continue; // eslint-disable-next-line no-await-in-loop const tids = await fs.readdir(path.join(pidDir, 'task')).catch(() => []); // eslint-disable-next-line no-restricted-syntax diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index ad9c698afd..21874c03c8 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -8,7 +8,6 @@ const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper' const hostStallService = require('../../ZelBack/src/services/hostStallService'); const { stuckIntervals } = config.hostStall; -const OWN_NAMESPACE = 'pid:[4026531836]'; describe('hostStallService tests', () => { let procRoot; @@ -19,14 +18,16 @@ describe('hostStallService tests', () => { * @param {string} pid * @param {string} comm * @param {Array<{tid: string, state: string, switches: number, wchan?: string, startTime?: string}>} threads - * @param {string} [namespace] + * @param {{inContainer?: boolean}} [options] In a PID namespace below FluxOS's. */ - function processWith(pid, comm, threads, namespace = OWN_NAMESPACE) { + function processWith(pid, comm, threads, { inContainer = false } = {}) { const dir = path.join(procRoot, pid); fs.rmSync(dir, { recursive: true, force: true }); - fs.mkdirSync(path.join(dir, 'ns'), { recursive: true }); + fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, 'comm'), `${comm}\n`); - fs.symlinkSync(namespace, path.join(dir, 'ns', 'pid')); + // its id in FluxOS's namespace, then in each namespace below; no ns/ link, + // which the kernel refuses to an account that is not the process's + fs.writeFileSync(path.join(dir, 'status'), `Name:\t${comm}\nNSpid:\t${pid}${inContainer ? '\t1' : ''}\n`); threads.forEach(({ tid, state, switches, wchan = 'kvm_async_pf_task_wait_schedule', startTime = '1000', }) => { @@ -51,8 +52,10 @@ describe('hostStallService tests', () => { beforeEach(() => { procRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-')); + // FluxOS may follow its own ns/ link; another account's, it may not fs.mkdirSync(path.join(procRoot, 'self', 'ns'), { recursive: true }); - fs.symlinkSync(OWN_NAMESPACE, path.join(procRoot, 'self', 'ns', 'pid')); + fs.symlinkSync('pid:[4026531836]', path.join(procRoot, 'self', 'ns', 'pid')); + fs.writeFileSync(path.join(procRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); paths = { procRoot, recordFile: path.join(procRoot, 'hostStall.json'), bootIdPath: path.join(procRoot, 'boot_id') }; bootIs('boot-one'); }); @@ -153,7 +156,7 @@ describe('hostStallService tests', () => { }); it('does not read a dependency running in an app container', async () => { - processWith('400', 'mongod', [{ tid: '401', state: 'D', switches: 3 }], 'pid:[4026532999]'); + processWith('400', 'mongod', [{ tid: '401', state: 'D', switches: 3 }], { inContainer: true }); await readTimes(stuckIntervals + 1); From 5fc8fc1e71365cefbfb9cb3a6c81105fc948ae37 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 09:06:25 +0100 Subject: [PATCH 225/369] fix(apps): a stall's reason names no wait point the kernel withholds The kernel writes 0 for the wait point of another account's thread, so a FluxOS that is not root reads it as unknown. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 4 +++- tests/unit/hostStallService.test.js | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 274df14c94..6b12bb13a9 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -78,8 +78,10 @@ async function readThread(taskDir) { const switches = status.split('\n') .filter((line) => line.startsWith('voluntary_ctxt_switches') || line.startsWith('nonvoluntary_ctxt_switches')) .reduce((sum, line) => sum + Number(line.split(/\s+/)[1]), 0); + // the kernel writes 0 for a wait point it does not show this account + const waitPoint = (wchan ?? '').trim(); return { - state: fields[0], startTime: fields[19], switches, wchan: (wchan ?? '').trim() || 'unknown', + state: fields[0], startTime: fields[19], switches, wchan: waitPoint && waitPoint !== '0' ? waitPoint : 'unknown', }; } diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 21874c03c8..4b571f4bb6 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -278,6 +278,14 @@ describe('hostStallService tests', () => { }); }); + it('names no wait point the kernel does not show', async () => { + processWith('700', 'dockerd', [{ tid: '701', state: 'D', switches: 9, wchan: '0' }]); + + await readTimes(2); + + expect(fluxNetworkHelper.getPlacementHold()).to.include('dockerd thread 701 has been held in the kernel at unknown'); + }); + it('reads a thread whose command name holds spaces and parentheses', async () => { processWith('500', 'syncthing', []); const taskDir = path.join(procRoot, '500', 'task', '501'); From 398ddb77b6c9d55e18611993fff5e1cd284dcedf Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 19:29:04 +0100 Subject: [PATCH 226/369] fix(apps): a stall reading under way when the service stops keeps nothing it read stop() advances a generation, and a reading begun before it discards what it read instead of storing it and acting on it; each reading reads the /proc it started with. A reading that outlived a stop otherwise replaced the next run's tracked threads with its own, resetting the count a stall is judged on. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 18 ++++++++++++----- tests/unit/hostStallService.test.js | 25 ++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 6b12bb13a9..58fd4dfb9e 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -40,6 +40,9 @@ const STUCK_INTERVALS = config.hostStall.stuckIntervals; */ let tracked = new Map(); let readTimer = null; +// Advanced by stop(), so a reading still under way when the service stops +// neither keeps what it read nor acts on it. +let generation = 0; /** Called once the node is out of service for a host stall. */ const tripListeners = []; const DEFAULT_PATHS = { @@ -102,15 +105,16 @@ function namespaceDepth(status) { /** * Every thread of every dependency process in this PID namespace. + * @param {string} procRoot The /proc this reading reads, fixed when it starts. * @returns {Promise>} */ -async function readDependencyThreads() { - const ownDepth = namespaceDepth(await readOrNull(path.join(paths.procRoot, 'self', 'status'))); - const pids = (await fs.readdir(paths.procRoot)).filter((entry) => /^\d+$/.test(entry)); +async function readDependencyThreads(procRoot) { + const ownDepth = namespaceDepth(await readOrNull(path.join(procRoot, 'self', 'status'))); + const pids = (await fs.readdir(procRoot)).filter((entry) => /^\d+$/.test(entry)); const threads = []; // eslint-disable-next-line no-restricted-syntax for (const pid of pids) { - const pidDir = path.join(paths.procRoot, pid); + const pidDir = path.join(procRoot, pid); // eslint-disable-next-line no-await-in-loop const comm = (await readOrNull(path.join(pidDir, 'comm')))?.trim(); // eslint-disable-next-line no-continue @@ -244,7 +248,10 @@ async function act() { */ async function readOnce(options = {}) { usePaths(options); - tracked = carryForward(await readDependencyThreads()); + const startedIn = generation; + const threads = await readDependencyThreads(paths.procRoot); + if (startedIn !== generation) return; + tracked = carryForward(threads); await act(); } @@ -272,6 +279,7 @@ async function start(options = {}) { /** Take no more readings and forget every thread. */ function stop() { + generation += 1; clearTimeout(readTimer); readTimer = null; tracked = new Map(); diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 4b571f4bb6..20afae90b5 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -68,6 +68,31 @@ describe('hostStallService tests', () => { sinon.restore(); }); + // A reading begun before a stop that ends after the next start's readings would + // replace what they have counted with what it read. + it('keeps nothing from a reading still under way when it stops', async () => { + const otherRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-other-')); + fs.mkdirSync(path.join(otherRoot, 'self')); + fs.writeFileSync(path.join(otherRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); + let finishReading = null; + const readdir = fs.promises.readdir.bind(fs.promises); + sinon.stub(fs.promises, 'readdir').callsFake((dir, ...rest) => (dir === otherRoot + ? new Promise((resolve) => { finishReading = () => resolve([]); }) + : readdir(dir, ...rest))); + stuckMongod(); + const underWay = hostStallService.readOnce({ ...paths, procRoot: otherRoot }); + while (!finishReading) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop + + hostStallService.stop(); + await readTimes(2); + finishReading(); + await underWay; + await readTimes(stuckIntervals - 1); + + expect(outOfService(), 'a reading begun before the stop reset what came after it').to.equal(true); + fs.rmSync(otherRoot, { recursive: true, force: true }); + }); + it('holds the node back once a thread stays in D without running for one interval', async () => { stuckMongod(); await read(); From f1349cc4e5a308c7bc61f1e2e26d1c708a9f10e5 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 15:50:47 +0100 Subject: [PATCH 227/369] test(harness): 3304 starts the joining node once every peer can answer it, and a refusal makes a run prove nothing Co-Authored-By: Claude Opus 5.5 --- ...a-sync-answer-the-node-cannot-record-is-not-counted.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js index 484f2a5f73..5b97e5666c 100644 --- a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js +++ b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js @@ -29,7 +29,7 @@ const JOINING = NODES - 1; // what the peers hold when the node joins: more events than one message carries const ANSWERED_EVENTS = ANSWER_BATCH + 1; // a peer's request ended for any of these reasons is replaced too, and is not this -const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified']; +const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified', 'sync:refused']; function step(what) { console.log(`# 3304 ${new Date().toISOString()} ${what}`); @@ -78,6 +78,12 @@ FAILURES.forEach(({ what, arm }) => describe(`a sync answer the node cannot reco timeout: 180000, interval: 2000, label: 'the peers hold more running-app events than one message carries', }); eventsHeld = (await dbClient(1).appStateEvents({ type: 'apprunning' })).length; + // Every peer answers before the node joins: a peer still syncing refuses it, + // and asking a third peer then says nothing about the one replaced. + await Promise.all(env.clients.slice(0, JOINING).map((client, i) => waitFor( + () => client.getEventBuffer().some((e) => e.event === 'orchestrator:stateChanged' && e.data?.to === 'READY'), + { timeout: 300000, interval: 1000, label: `node ${i} answers a state sync` }, + ))); step(`arming ${what}`); release = await arm(JOINING + 1); From 495c717c890f87c2a74f882faf657703bfd0b166 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 15:57:25 +0100 Subject: [PATCH 228/369] test(harness): 3304 does not count the joining node's own refusals as another ending sync:refused is published by a node refusing a peer's request, which the joining node does while it is not yet authoritative; it says nothing about the peers it asks. Co-Authored-By: Claude Opus 5.5 --- .../3304-a-sync-answer-the-node-cannot-record-is-not-counted.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js index 5b97e5666c..c9e5bd0ff5 100644 --- a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js +++ b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js @@ -29,7 +29,7 @@ const JOINING = NODES - 1; // what the peers hold when the node joins: more events than one message carries const ANSWERED_EVENTS = ANSWER_BATCH + 1; // a peer's request ended for any of these reasons is replaced too, and is not this -const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified', 'sync:refused']; +const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified']; function step(what) { console.log(`# 3304 ${new Date().toISOString()} ${what}`); From af562868bff6626d09ba9dacd8b5745100df5af4 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 16:27:06 +0100 Subject: [PATCH 229/369] fix(apps): a host stall stops every app at once and kills what still runs after one shared window The unload asks docker to stop every container of every app at once, waits hostStall.stopWindowMs (60 s) for all of them together, and kills through its cgroups whatever of each app can still run. Only then does each app get its normal removal, which tells the network. A docker that cannot act costs the window once, not one removal deadline per app. hostStall:appsStopped reports what of each app can still run once that is done. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 6 +- ZelBack/src/services/appLifecycle/appKill.js | 10 ++ .../appMonitoring/nodeStatusMonitor.js | 58 +++++++++--- test-infra/runner/framework/node-client.js | 1 + tests/unit/appKill.test.js | 11 +++ tests/unit/globalconfig/default.js | 1 + tests/unit/nodeStatusMonitor.test.js | 94 +++++++++++++++---- 7 files changed, 148 insertions(+), 33 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index 5edf694e9b..df3e05bc7e 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -42,10 +42,14 @@ module.exports = { }, // A thread of a process FluxOS depends on that stays in the kernel without // running for one interval holds the node back from new placements, and for - // stuckIntervals intervals takes it out of service until it reboots. + // stuckIntervals intervals takes it out of service until it reboots. Its apps + // are then asked to stop all at once, and whatever of them can still run after + // stopWindowMs is killed: longer than the slowest removal of a component seen on + // the fleet (39 s), which stops it first. hostStall: { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, + stopWindowMs: 60 * 1000, }, database: { url: '127.0.0.1', diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index b403b85ac5..3c022623d1 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -93,6 +93,15 @@ async function containersOnDisk() { return containers; } +/** + * The names of the app's containers, read from docker's records on disk. + * @param {string} appName + * @returns {Promise} + */ +async function containersOf(appName) { + return (await containersOnDisk()).filter(({ name }) => belongsTo(name, appName)).map(({ name }) => name); +} + /** * The scope units of the app's containers, whether in its slice or not. * @param {string} appName @@ -196,6 +205,7 @@ function reset() { } module.exports = { + containersOf, kill, stillRunnable, appsOnDisk, diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index f78d2c503e..09e2fbdff8 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -10,6 +10,7 @@ const nodeConfirmationService = require('../nodeConfirmationService'); const networkStateService = require('../networkStateService'); const hostStallService = require('../hostStallService'); const appKill = require('../appLifecycle/appKill'); +const dockerService = require('../dockerService'); const { RemovalOutcome } = require('../utils/removalOutcome'); const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); const fluxEventBus = require('../utils/fluxEventBus'); @@ -22,6 +23,7 @@ let unloadingForHostStall = false; let deps = null; const REMOVAL_DEADLINE_MS = config.fluxapps.appRemovalDeadlineMs; +const STOP_WINDOW_MS = config.hostStall.stopWindowMs; const STILL_REMOVING = 'stillRemoving'; /** Apps whose removal started here and has not returned, deadline or not. */ const removalsRunning = new Set(); @@ -104,16 +106,30 @@ async function removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, reason } } +/** + * Ask docker to stop every container of the app, on its normal grace. + * @param {string} name + * @returns {Promise} + */ +async function stopApp(name) { + const containers = await appKill.containersOf(name); + await Promise.allSettled(containers.map((container) => dockerService.appDockerStop(container))); +} + /** * Remove every app from a node whose host is stalled, and make sure none can - * run again. Each app gets its normal removal, which stops its containers before - * it tells the network, within REMOVAL_DEADLINE_MS. Whatever of it can still run - * afterwards is killed through its cgroups, and checked: an app whose every - * process is gone or has the kill pending can never run its code again. + * run again before the network is told. + * + * Every app is asked to stop at once, and all of them share one STOP_WINDOW_MS: + * a docker that can still deliver a stop signal stops each app cleanly, and one + * that cannot costs the window once, not once per app. Whatever of each app can + * still run after the window is killed through its cgroups and checked: an app + * whose every process is gone or has the kill pending can never run its code + * again. Then each app gets its normal removal, which tells the network, within + * REMOVAL_DEADLINE_MS; nothing of it runs by then, so the removals take turns. * * The apps are read from the database and from what is on disk, so an app the - * database cannot list is still found. Apps are not spaced out: every minute a - * stalled node keeps one is a minute its standby waits. + * database cannot list is still found. * @param {Function} installedAppsFn * @param {Function} removeAppLocallyFn */ @@ -124,18 +140,17 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { const listed = await withinDeadline(installedAppsFn(), REMOVAL_DEADLINE_MS).catch(() => null); const installed = listed?.status === 'success' ? listed.data.map((app) => app.name) : []; const names = [...new Set([...installed, ...await appKill.appsOnDisk()])]; - const canBroadcast = nodeConfirmationService.canSendMessages(); + log.warn(`monitorNodeStatus - host stall: stopping ${names.length} applications, ${STOP_WINDOW_MS / 1000}s between them`); + await withinDeadline(Promise.allSettled(names.map(stopApp)), STOP_WINDOW_MS); + + const stopped = new Map(); // eslint-disable-next-line no-restricted-syntax for (const name of names) { - log.warn(`monitorNodeStatus - Application ${name} going to be removed: host stall`); - // eslint-disable-next-line no-await-in-loop - const outcome = await removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) - .catch(() => RemovalOutcome.FAILED); // eslint-disable-next-line no-await-in-loop let runnable = await appKill.stillRunnable(name); const killed = runnable.length > 0; if (killed) { - log.error(`monitorNodeStatus - Application ${name} still runs after its removal (${outcome}), killing it`); + log.error(`monitorNodeStatus - Application ${name} still runs after being asked to stop, killing it`); // eslint-disable-next-line no-await-in-loop await appKill.kill(name); // eslint-disable-next-line no-await-in-loop @@ -145,10 +160,25 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { } else { log.warn(`monitorNodeStatus - Application ${name} can no longer run`); } - } else if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + } + stopped.set(name, { killed, runnable }); + } + // runnable: the processes of each app that can still run its code, empty when none can + fluxEventBus.publish('hostStall:appsStopped', { + apps: names.map((name) => ({ app: name, ...stopped.get(name) })), + }); + + const canBroadcast = nodeConfirmationService.canSendMessages(); + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + log.warn(`monitorNodeStatus - Application ${name} going to be removed: host stall`); + // eslint-disable-next-line no-await-in-loop + const outcome = await removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) + .catch(() => RemovalOutcome.FAILED); + const { killed, runnable } = stopped.get(name); + if (!runnable.length && outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { log.warn(`monitorNodeStatus - Application ${name} was not removed (${outcome}), and can no longer run`); } - // runnable: the processes of the app that can still run its code, empty when none can fluxEventBus.publish('hostStall:appUnloaded', { app: name, removal: outcome, killed, runnable, }); diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 61d985f67c..193e69bb92 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -313,6 +313,7 @@ export function nodeClient(nodeNum) { 'placementHold:set', 'placementHold:cleared', 'hostStall:appUnloaded', + 'hostStall:appsStopped', 'network:apprunning', 'network:appinstalling', 'network:appinstallingerror', diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index 1b6e57b7a0..b5036170b8 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -159,4 +159,15 @@ describe('appKill tests', () => { expect(await appKill.appsOnDisk()).to.have.members(['sliced-app', 'composed', 'single']); }); }); + + describe('containersOf', () => { + it('names every container of the app from docker\'s records, and no other app\'s', async () => { + record(1, 'fluxweb_composed'); + record(2, 'fluxdb_composed'); + record(3, 'fluxweb_composedtoo'); + record(4, 'fluxcomposed'); + + expect(await appKill.containersOf('composed')).to.have.members(['fluxweb_composed', 'fluxdb_composed', 'fluxcomposed']); + }); + }); }); diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 5c4e6b90cd..bc7a836e61 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -32,6 +32,7 @@ module.exports = { hostStall: { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, + stopWindowMs: 60 * 1000, }, database: { url: database, diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 974661ddad..5849f79338 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -25,6 +25,7 @@ describe('nodeStatusMonitor tests', () => { }, config: { fluxapps: { appRemovalDeadlineMs: 300000 }, + hostStall: { stopWindowMs: 60000 }, database: { appsglobal: { database: 'globalapps', @@ -118,9 +119,11 @@ describe('nodeStatusMonitor tests', () => { describe('removal while out of service', () => { const DEADLINE = 300000; + const STOP_WINDOW = 60000; let clock; let monitor; let appKill; + let dockerService; let onTrip; let dos; let bus; @@ -133,11 +136,17 @@ describe('nodeStatusMonitor tests', () => { bus = { publish: sinon.stub() }; appKill = { appsOnDisk: sinon.stub().resolves([]), + containersOf: sinon.stub().callsFake(async (name) => [`flux${name}`]), stillRunnable: sinon.stub().resolves([]), kill: sinon.stub().resolves([]), }; + dockerService = { appDockerStop: sinon.stub().resolves('stopped') }; monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { - config: { fluxapps: { appRemovalDeadlineMs: DEADLINE, nodeMonitorRemovalDelayMs: 0 }, database: { appsglobal: { collections: {} } } }, + config: { + fluxapps: { appRemovalDeadlineMs: DEADLINE, nodeMonitorRemovalDelayMs: 0 }, + hostStall: { stopWindowMs: STOP_WINDOW }, + database: { appsglobal: { collections: {} } }, + }, '../dbHelper': {}, // the spacing between apps passes, and the wait before the next pass never ends '../serviceHelper': { delay: sinon.stub().callsFake((ms) => (ms === 0 ? Promise.resolve() : never())) }, @@ -150,6 +159,7 @@ describe('nodeStatusMonitor tests', () => { }, '../hostStallService': { onTrip: (listener) => { onTrip = listener; } }, '../appLifecycle/appKill': appKill, + '../dockerService': dockerService, '../utils/fluxEventBus': bus, '../../lib/log': logStub, }); @@ -162,23 +172,74 @@ describe('nodeStatusMonitor tests', () => { it('publishes an app its removal removed, with nothing left to run', async () => { await monitor.unloadForHostStall(installed('gone'), sinon.stub().resolves('removed')); - sinon.assert.calledOnceWithExactly(bus.publish, 'hostStall:appUnloaded', { + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appUnloaded', { app: 'gone', removal: 'removed', killed: false, runnable: [], }); }); - it('publishes an app it killed after its removal ran out, and what of it can still run', async () => { + it('publishes an app it killed, and what of it can still run, once it has stopped every app', async () => { appKill.stillRunnable.onFirstCall().resolves(['41', '42']).onSecondCall().resolves(['42']); const unload = monitor.unloadForHostStall(installed('stuck'), sinon.stub().callsFake(never)); - await clock.tickAsync(DEADLINE); + await clock.tickAsync(STOP_WINDOW + DEADLINE); await unload; - sinon.assert.calledOnceWithExactly(bus.publish, 'hostStall:appUnloaded', { + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { + apps: [{ app: 'stuck', killed: true, runnable: ['42'] }], + }); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appUnloaded', { app: 'stuck', removal: 'timedOut', killed: true, runnable: ['42'], }); }); + it('asks docker to stop every container of every app at once', async () => { + appKill.containersOf.withArgs('one').resolves(['fluxweb_one', 'fluxdb_one']); + dockerService.appDockerStop.callsFake(never); + + const unload = monitor.unloadForHostStall(installed('one', 'two'), sinon.stub().resolves('removed')); + await clock.tickAsync(0); + + expect(dockerService.appDockerStop.getCalls().map((call) => call.args[0])).to.have.members(['fluxweb_one', 'fluxdb_one', 'fluxtwo']); + await clock.tickAsync(STOP_WINDOW); + await unload; + }); + + it('waits one stop window for every app together, then kills what can still run', async () => { + dockerService.appDockerStop.callsFake(never); + appKill.stillRunnable.callsFake(async (name) => (appKill.kill.calledWith(name) ? [] : ['41'])); + + const unload = monitor.unloadForHostStall(installed('a', 'b', 'c'), sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW - 1); + sinon.assert.notCalled(appKill.kill); + await clock.tickAsync(1); + await unload; + + expect(appKill.kill.getCalls().map((call) => call.args[0])).to.deep.equal(['a', 'b', 'c']); + }); + + it('goes on as soon as docker has stopped every app, without waiting out the window', async () => { + const remove = sinon.stub().resolves('removed'); + + await monitor.unloadForHostStall(installed('a', 'b'), remove); + + sinon.assert.calledTwice(remove); + sinon.assert.notCalled(appKill.kill); + }); + + it('removes no app, and so tells the network of none, until every app has stopped or been killed', async () => { + dockerService.appDockerStop.callsFake(never); + appKill.stillRunnable.callsFake(async (name) => (appKill.kill.calledWith(name) ? [] : ['41'])); + const remove = sinon.stub().resolves('removed'); + + const unload = monitor.unloadForHostStall(installed('a', 'b'), remove); + await clock.tickAsync(STOP_WINDOW - 1); + sinon.assert.notCalled(remove); + await clock.tickAsync(1); + await unload; + + sinon.assert.callOrder(appKill.kill.withArgs('b'), remove.withArgs('a')); + }); + it('removes every app once the host stalls, starting from the trip', async () => { const remove = sinon.stub().resolves('removed'); monitor.initialize(installed('one', 'two'), remove); @@ -190,20 +251,26 @@ describe('nodeStatusMonitor tests', () => { sinon.assert.notCalled(appKill.kill); }); - it('kills an app whose removal does not return within the deadline, and moves on', async () => { + it('moves past a removal that does not return within the deadline', async () => { const remove = sinon.stub(); remove.withArgs('stuck').callsFake(never); remove.withArgs('next').resolves('removed'); - appKill.stillRunnable.withArgs('stuck').onFirstCall().resolves(['41']).onSecondCall().resolves([]); const unload = monitor.unloadForHostStall(installed('stuck', 'next'), remove); await clock.tickAsync(DEADLINE - 1); - sinon.assert.notCalled(appKill.kill); + sinon.assert.neverCalledWith(remove, 'next'); await clock.tickAsync(1); await unload; - sinon.assert.calledOnceWithExactly(appKill.kill, 'stuck'); sinon.assert.calledWith(remove, 'next'); + }); + + it('says an app it killed can no longer run', async () => { + appKill.stillRunnable.onFirstCall().resolves(['41']).onSecondCall().resolves([]); + + await monitor.unloadForHostStall(installed('stuck'), sinon.stub().resolves('removed')); + + sinon.assert.calledOnceWithExactly(appKill.kill, 'stuck'); sinon.assert.calledWithMatch(logStub.warn, /stuck can no longer run/); }); @@ -224,15 +291,6 @@ describe('nodeStatusMonitor tests', () => { sinon.assert.neverCalledWithMatch(logStub.warn, /can no longer run/); }); - it('kills an app whose removal failed while its processes can still run', async () => { - const remove = sinon.stub().resolves('failed'); - appKill.stillRunnable.onFirstCall().resolves(['41']).onSecondCall().resolves([]); - - await monitor.unloadForHostStall(installed('app'), remove); - - sinon.assert.calledOnceWithExactly(appKill.kill, 'app'); - }); - it('names the processes a kill could not stop', async () => { appKill.stillRunnable.resolves(['41', '42']); From e72b7df769e99cad008a56d7ae0d9276aa068b0f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 16:27:06 +0100 Subject: [PATCH 230/369] test(harness): 3303, a node holding several apps has every one stopped within one stop window of a stall Co-Authored-By: Claude Opus 5.5 --- ...lled-host-takes-the-node-out-of-service.js | 89 +++++++++++++++++-- 1 file changed, 82 insertions(+), 7 deletions(-) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 10a3831946..bb33b8f781 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -21,11 +21,14 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // // A thread held for one reading interval holds the node back from new placements, // and the hold lifts when the thread runs again. Held for stuckIntervals intervals, -// the node goes out of service until it reboots, and removes every app within a -// deadline each. On a frozen disk docker still delivers the stop signal, so the -// app is stopped though its removal runs out. With docker frozen as well -// (systemd-control.freezeUnit), docker cannot act at all, and the app is killed -// through its slice without docker. +// the node goes out of service until it reboots: every app is asked to stop at +// once, whatever of them can still run after one shared stop window is killed +// through its slice, and then each is removed within a deadline. On a frozen disk +// docker still delivers the stop signal, so the app is stopped though its removal +// runs out. With docker frozen as well (systemd-control.freezeUnit), docker +// cannot act at all, and the app is killed without docker. A node holding several +// apps has every one of them stopped within that one window, long before even one +// removal's deadline. // // The nodes run systemd, so docker uses its systemd cgroup driver and every app's // containers live in the app's own slice, as on a production node. The node the @@ -37,7 +40,10 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const READ_INTERVAL_MS = 5 * 1000; const STUCK_INTERVALS = 6; -const REMOVAL_DEADLINE_MS = 20 * 1000; +const REMOVAL_DEADLINE_MS = 45 * 1000; +const STOP_WINDOW_MS = 10 * 1000; +// what the node takes beyond the window to check and kill every app +const KILL_SLACK_MS = 10 * 1000; const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; const OWNER = 'hostStall'; const ARCANE = [0, 1, 2]; @@ -88,7 +94,7 @@ describe('a stalled host takes the node out of service', function () { legacyNodes: [LEGACY_UNPRIVILEGED], unprivilegedNodes: [LEGACY_UNPRIVILEGED], configOverrides: { - hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS }, + hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS, stopWindowMs: STOP_WINDOW_MS }, fluxapps: { appRemovalDeadlineMs: REMOVAL_DEADLINE_MS }, }, }); @@ -258,4 +264,73 @@ describe('a stalled host takes the node out of service', function () { expect(await sliceProcessesThatCanRun(killContainer, killedName), 'a process of the app can still run').to.deep.equal([]); }); }); + + describe('a stall past the threshold, with docker unable to act at all, on a node holding several apps', function () { + const stamp = Date.now(); + const names = [1, 2, 3].map((n) => `e2ehoststallmany${n}${stamp}`); + let manyClient; + let manyContainer; + let anchor; + let runningBefore; + let trippedAt; + let stopped; + let runnableAfterWindow; + + before(async function () { + this.timeout(480000); + const index = ARCANE.find((i) => i !== node); + manyClient = env.clients[index]; + ({ container: manyContainer } = manyClient); + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + // eslint-disable-next-line no-await-in-loop + await installOn(env, name, [index]); + // eslint-disable-next-line no-await-in-loop + await waitForUp(manyClient, name, `${name} is up`); + } + anchor = manyClient.getLastEventId(); + step(`freezing node ${index}, which holds ${names.length} apps`); + await freezeDocker(manyContainer); + await freezeUnit(manyContainer, DOCKERD_UNIT); + runningBefore = await Promise.all(names.map((name) => sliceProcessesThatCanRun(manyContainer, name))); + const trip = await manyClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + stopped = (await manyClient.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + // read once the window and the kill are over, and before even one removal's deadline + runnableAfterWindow = trippedAt + ? await Promise.all(names.map((name) => sliceProcessesThatCanRun(manyContainer, name))) + : null; + step('read every app after the stop window'); + }); + + after(async function () { + if (manyClient) await thawEveryNode([manyClient]); + }); + + it('held docker in the kernel, unable to answer, while every app could run', async function () { + const stuck = await stuckThreads(manyContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + const answer = await execInContainer(manyContainer, 'timeout 5 docker version'); + expect(answer.exitCode, 'docker still answers, so it could still stop the apps itself').to.not.equal(0); + runningBefore.forEach((running, i) => expect(running, `${names[i]} was not running, so there was nothing to stop`).to.not.be.empty); + }); + + it('takes the node out of service', function () { + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('stops every app within one stop window, before even one removal\'s deadline', function () { + expect(STOP_WINDOW_MS + KILL_SLACK_MS, 'fixture: the window and the kill do not fit inside one removal\'s deadline').to.be.below(REMOVAL_DEADLINE_MS); + runnableAfterWindow.forEach((runnable, i) => expect(runnable, `${names[i]} can still run one stop window after the node went out of service`).to.deep.equal([])); + }); + + it('says every app was stopped, and that nothing of any can run', function () { + expect(stopped, 'the node never said its apps were stopped').to.not.equal(null); + expect(stopped.apps.map((a) => a.app), 'the apps the node stopped').to.include.members(names); + stopped.apps.filter((a) => names.includes(a.app)) + .forEach((a) => expect(a.runnable, `the node reports a process of ${a.app} that can still run`).to.deep.equal([])); + }); + }); }); From bf05054666b401012295aa46beffe174e9b0e95f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 19:47:20 +0100 Subject: [PATCH 231/369] fix(apps): a node runs apps only on cgroup v2 with cgroup.kill under Docker's systemd cgroup driver A node missing any of the three holds itself out of service at boot and names what is missing. App slices, CPU burst and the host-stall kill rely on all three, so each uses them without a fallback. Co-Authored-By: Claude Opus 5.5 --- .../services/appManagement/appInspector.js | 12 +--- ZelBack/src/services/dockerService.js | 8 +-- ZelBack/src/services/fluxNetworkHelper.js | 35 ++++++++++ ZelBack/src/services/serviceManager.js | 5 +- ZelBack/src/services/utils/cpuBurstHelper.js | 64 ++----------------- tests/unit/appInspector.test.js | 8 +-- tests/unit/cpuBurstHelper.test.js | 64 +++++-------------- tests/unit/dockerService.test.js | 14 +--- tests/unit/fluxNetworkHelper.test.js | 56 ++++++++++++++++ 9 files changed, 125 insertions(+), 141 deletions(-) diff --git a/ZelBack/src/services/appManagement/appInspector.js b/ZelBack/src/services/appManagement/appInspector.js index fa871845de..f08ec65b21 100644 --- a/ZelBack/src/services/appManagement/appInspector.js +++ b/ZelBack/src/services/appManagement/appInspector.js @@ -318,12 +318,9 @@ function extractSample(stats) { // space. `docker stats` subtracts it, so every consumer that reports a // container's memory subtracts it too, and without this figure none of them // can: they subtract zero and report a number that climbs with disk reads. - // cgroup v2 names it inactive_file and v1 names it cache; they are not the - // same counter (v1's is the whole page cache, v2's the reclaimable part) but - // docker's own CLI reads them from the same slot, so this does too. Null - // where neither is present, which leaves the consumer exactly where it was. - memoryCache: stats.memory_stats?.stats?.inactive_file - ?? stats.memory_stats?.stats?.cache ?? null, + // cgroup v2 names it inactive_file. Null where it is absent, which leaves + // the consumer exactly where it was. + memoryCache: stats.memory_stats?.stats?.inactive_file ?? null, ioRead: sumBlkio('read'), ioWrite: sumBlkio('write'), // EVERY interface, not just eth0. appNetworkLinker connects a container to @@ -362,9 +359,6 @@ function expandSample(sample) { cpu_usage: { total_usage: sample.cpuTotalBefore }, system_cpu_usage: sample.cpuSystemBefore, }, - // Reported under the cgroup v2 name whichever key it was read from: a - // consumer's `inactive_file ?? cache` chain then settles on the first, and - // the value is the same either way. memory_stats: { usage: sample.memoryUsage, limit: sample.memoryLimit, diff --git a/ZelBack/src/services/dockerService.js b/ZelBack/src/services/dockerService.js index 840f49864b..77bee91b1c 100644 --- a/ZelBack/src/services/dockerService.js +++ b/ZelBack/src/services/dockerService.js @@ -1170,7 +1170,7 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS const appOwner = fullAppSpecs?.owner || null; const burstEligible = appOwner && cpuBurstHelper.isEnterpriseOwner(appOwner) - && await cpuBurstHelper.isCpuBurstSupported(); + && cpuBurstHelper.isCpuBurstEnabled(); const burstLabels = burstEligible ? { 'flux.burst.eligible': 'true', @@ -1240,12 +1240,8 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS // eslint-disable-next-line no-use-before-define const dockerInfoResp = await dockerInfo(); log.info(dockerInfoResp); - // The app's own slice is a systemd unit, so it is the parent only under the - // systemd cgroup driver; under another the container keeps docker's default. const slice = appSlice.sliceName(appName); - if (dockerInfoResp.CgroupDriver === 'systemd' && slice) { - options.HostConfig.CgroupParent = slice; - } + if (slice) options.HostConfig.CgroupParent = slice; const driverStatus = dockerInfoResp.DriverStatus; const backingFs = driverStatus.find((status) => status[0] === 'Backing Filesystem'); // d_type must be true for overlay, docker would not work if not if (backingFs && backingFs[1] === 'xfs') { diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 73b1000424..3e3cc40c17 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -99,6 +99,7 @@ const StickyDosOwner = Object.freeze({ NODEJS_FLOOR: 'nodejsFloor', DOCKER_FLOOR: 'dockerFloor', MONGODB_FLOOR: 'mongodbFloor', + CGROUP_FLOOR: 'cgroupFloor', HOST_STALL: 'hostStall', }); @@ -1461,6 +1462,39 @@ function checkDockerVersionAllowed(dockerVersion) { return false; } +/** + * Whether this node's cgroups are the ones FluxOS runs apps in: cgroup v2, + * a kernel that can kill a whole cgroup (cgroup.kill, Linux 5.14), and Docker + * placing containers through systemd. App slices, CPU burst and the kill a + * host stall makes all rely on the three. + * + * Asked once, when Docker first answers at startup. The kernel's answers are + * read from the cgroup filesystem; init.scope is the cgroup systemd always + * creates, and cgroup.kill exists in every cgroup but the root. A driver that + * could not be read gives no verdict on the driver. + * @param {?string} cgroupDriver The cgroup driver Docker reports, or null when + * it could not be read. + * @param {string} [cgroupRoot] Where the cgroup filesystem is mounted. + * @returns {Promise} True unless a requirement is known to be unmet. + */ +async function checkCgroupsAllowed(cgroupDriver, cgroupRoot = '/sys/fs/cgroup') { + const exists = (file) => fs.access(path.join(cgroupRoot, file)).then(() => true, () => false); + const unmet = []; + if (!await exists('cgroup.controllers')) unmet.push('cgroup v2'); + else if (!await exists('init.scope/cgroup.kill')) unmet.push('cgroup.kill (Linux 5.14 or later)'); + if (cgroupDriver === null) { + log.error('checkCgroupsAllowed - Docker did not report its cgroup driver, no verdict on it'); + } else if (cgroupDriver !== 'systemd') { + unmet.push(`Docker's systemd cgroup driver (found ${cgroupDriver})`); + } + if (!unmet.length) return true; + setStickyDos( + StickyDosOwner.CGROUP_FLOOR, + `Cgroup Error. FluxOS requires cgroup v2, cgroup.kill and Docker's systemd cgroup driver; missing: ${unmet.join(', ')}. Fix the host and restart FluxOS`, + ); + return false; +} + /** * Whether the MongoDB this node runs meets the network minimum: the oldest * release FluxOS supports. @@ -3725,6 +3759,7 @@ module.exports = { checkFluxbenchVersionAllowed, checkNodeJsVersionAllowed, checkDockerVersionAllowed, + checkCgroupsAllowed, checkMongoDbVersionAllowed, checkMyFluxAvailability, adjustExternalIp, diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 4746a9b93d..cce78a1508 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -277,14 +277,15 @@ async function startFluxFunctions() { // DOS, so /flux/info names the version found and the version required. fluxNetworkHelper.checkMongoDbVersionAllowed(await dbHelper.getMongoDbVersion()); await dockerService.waitForDocker(); + const dockerInfo = await dockerService.dockerInfo().catch(() => null); // Where docker keeps its records, read while it answers, for a kill made // while it does not. - const dockerRoot = await dockerService.dockerInfo().then((info) => info?.DockerRootDir, () => null); - if (dockerRoot) appKill.usePaths({ dockerRoot }); + if (dockerInfo?.DockerRootDir) appKill.usePaths({ dockerRoot: dockerInfo.DockerRootDir }); // As soon as Docker answers, before the container firewall is applied: a // below-floor node stays up holding a sticky DOS, so /flux/info names the // version found and the version required. fluxNetworkHelper.checkDockerVersionAllowed(await dockerService.dockerVersion().then((version) => version.Version, () => null)); + await fluxNetworkHelper.checkCgroupsAllowed(dockerInfo?.CgroupDriver ?? null); // Before any work that a restart would cut short: a registration that // needs its kill timeout raised restarts FluxOS, and this waits for it. await fluxService.ensurePm2KillTimeout(); diff --git a/ZelBack/src/services/utils/cpuBurstHelper.js b/ZelBack/src/services/utils/cpuBurstHelper.js index c704fbe79d..0686a37079 100644 --- a/ZelBack/src/services/utils/cpuBurstHelper.js +++ b/ZelBack/src/services/utils/cpuBurstHelper.js @@ -6,8 +6,6 @@ const log = require('../../lib/log'); const enterpriseConfig = require('./enterpriseConfig'); const signatureVerifier = require('../signatureVerifier'); -let burstSupportCache = null; - /** * Checks if an app owner is in the enterprise app owners whitelist. An unknown policy * answers false: burst is a privilege, and one that cannot be verified is not granted. @@ -21,52 +19,12 @@ function isEnterpriseOwner(owner) { } /** - * Checks if the system supports CFS CPU burst (cgroups v2 + kernel >= 5.14). - * Caches the result after first check. - * @returns {Promise} + * Whether CPU burst is switched on. Every node FluxOS runs apps on has what it + * needs: cgroup v2 and Linux 5.14 (fluxNetworkHelper.checkCgroupsAllowed). + * @returns {boolean} */ -async function isCpuBurstSupported() { - if (burstSupportCache !== null) return burstSupportCache; - - try { - if (!config.cpuBurst?.enabled) { - burstSupportCache = false; - return false; - } - - // Check cgroups v2: /sys/fs/cgroup/cgroup.controllers must exist - try { - await fsp.access('/sys/fs/cgroup/cgroup.controllers'); - } catch { - log.warn('CPU burst: cgroups v2 not detected, burst disabled'); - burstSupportCache = false; - return false; - } - - // Check kernel version >= 5.14 for cpu.cfs_burst_us support - const release = await fsp.readFile('/proc/version', 'utf8'); - const match = release.match(/(\d+)\.(\d+)/); - if (!match) { - log.warn('CPU burst: unable to parse kernel version, burst disabled'); - burstSupportCache = false; - return false; - } - const major = parseInt(match[1], 10); - const minor = parseInt(match[2], 10); - if (major < 5 || (major === 5 && minor < 14)) { - log.warn(`CPU burst: kernel ${major}.${minor} < 5.14, burst disabled`); - burstSupportCache = false; - return false; - } - - burstSupportCache = true; - log.info('CPU burst: system supports CFS burst (cgroups v2, kernel >= 5.14)'); - return true; - } catch (error) { - log.warn(`CPU burst: capability check failed: ${error.message}`); - burstSupportCache = false; - return false; - } +function isCpuBurstEnabled() { + return config.cpuBurst?.enabled === true; } /** @@ -185,7 +143,7 @@ async function isBurstActive(pid) { */ async function applyBurst(pid, cpuCores, identifier) { if (!cpuCores || cpuCores <= 0) return false; - if (!await isCpuBurstSupported()) return false; + if (!isCpuBurstEnabled()) return false; const { burstUs } = calculateBurstParams(cpuCores); const ok = await setCpuBurst(pid, burstUs); @@ -198,20 +156,12 @@ async function applyBurst(pid, cpuCores, identifier) { return ok; } -/** - * Resets the burst support cache. Used in testing. - */ -function resetBurstSupportCache() { - burstSupportCache = null; -} - module.exports = { isEnterpriseOwner, - isCpuBurstSupported, + isCpuBurstEnabled, getCgroupBurstPath, calculateBurstParams, setCpuBurst, isBurstActive, applyBurst, - resetBurstSupportCache, }; diff --git a/tests/unit/appInspector.test.js b/tests/unit/appInspector.test.js index 68b9a7250b..e6bfb8fe2e 100644 --- a/tests/unit/appInspector.test.js +++ b/tests/unit/appInspector.test.js @@ -340,13 +340,7 @@ describe('appInspector tests', () => { expect(stored.memoryCache).to.equal(512); }); - it('takes the cgroup v1 name when the v2 one is absent', async () => { - const stored = await tickOnce({ stats: { cache: 300 } }); - - expect(stored.memoryCache).to.equal(300); - }); - - it('records no cache rather than a zero when the host reports neither', async () => { + it('records no cache rather than a zero when the host does not report it', async () => { // Zero would read as "nothing cached" and be silently subtracted; null says // the figure is unavailable, and leaves the consumer where it was before. const stored = await tickOnce({}); diff --git a/tests/unit/cpuBurstHelper.test.js b/tests/unit/cpuBurstHelper.test.js index 7c25f23b36..aa2fe4855f 100644 --- a/tests/unit/cpuBurstHelper.test.js +++ b/tests/unit/cpuBurstHelper.test.js @@ -12,13 +12,11 @@ describe('cpuBurstHelper tests', () => { let cpuBurstHelper; beforeEach(() => { - // Fresh require each time to reset the cache cpuBurstHelper = proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { './enterpriseConfig': { getEnterpriseAppOwners: () => ['16mzUh6byiQr7rnYQxKraDbeBPsEHYpSTW'], }, }); - cpuBurstHelper.resetBurstSupportCache(); }); afterEach(() => { @@ -153,51 +151,21 @@ describe('cpuBurstHelper tests', () => { }); }); - describe('isCpuBurstSupported', () => { - it('should return false when burst config is disabled', async () => { - const helper = proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { - config: { cpuBurst: { enabled: false, periodUs: 100000 } }, - }); - helper.resetBurstSupportCache(); - const result = await helper.isCpuBurstSupported(); - expect(result).to.be.false; + describe('isCpuBurstEnabled', () => { + const withBurst = (cpuBurst) => proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { + config: { cpuBurst }, }); - it('should return false when cgroups v2 is not present', async () => { - sinon.stub(fs.promises, 'access').rejects(new Error('ENOENT')); - cpuBurstHelper.resetBurstSupportCache(); - - const result = await cpuBurstHelper.isCpuBurstSupported(); - expect(result).to.be.false; + it('is on when the config switches it on', () => { + expect(withBurst({ enabled: true, periodUs: 100000 }).isCpuBurstEnabled()).to.equal(true); }); - it('should return false for kernel < 5.14', async () => { - sinon.stub(fs.promises, 'access').resolves(); - sinon.stub(fs.promises, 'readFile').resolves('Linux version 5.4.0-generic'); - cpuBurstHelper.resetBurstSupportCache(); - - const result = await cpuBurstHelper.isCpuBurstSupported(); - expect(result).to.be.false; + it('is off when the config switches it off', () => { + expect(withBurst({ enabled: false, periodUs: 100000 }).isCpuBurstEnabled()).to.equal(false); }); - it('should return true for kernel >= 5.14 with cgroups v2', async () => { - sinon.stub(fs.promises, 'access').resolves(); - sinon.stub(fs.promises, 'readFile').resolves('Linux version 6.1.0-generic'); - cpuBurstHelper.resetBurstSupportCache(); - - const result = await cpuBurstHelper.isCpuBurstSupported(); - expect(result).to.be.true; - }); - - it('should cache the result on subsequent calls', async () => { - sinon.stub(fs.promises, 'access').resolves(); - const readStub = sinon.stub(fs.promises, 'readFile').resolves('Linux version 6.1.0-generic'); - cpuBurstHelper.resetBurstSupportCache(); - - await cpuBurstHelper.isCpuBurstSupported(); - await cpuBurstHelper.isCpuBurstSupported(); - // readFile called once for kernel version (cached after that) - expect(readStub.callCount).to.equal(1); + it('is off when the config does not mention it', () => { + expect(withBurst(undefined).isCpuBurstEnabled()).to.equal(false); }); }); @@ -307,12 +275,14 @@ describe('cpuBurstHelper tests', () => { expect(result).to.be.false; }); - it('should return false when isCpuBurstSupported is false', async () => { - sinon.stub(fs.promises, 'access').rejects(new Error('ENOENT')); - cpuBurstHelper.resetBurstSupportCache(); + it('applies nothing while burst is switched off', async () => { + const writeStub = sinon.stub(fs.promises, 'writeFile').resolves(); + const helper = proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { + config: { cpuBurst: { enabled: false, periodUs: 100000 } }, + }); - const result = await cpuBurstHelper.applyBurst(12345, 2.0); - expect(result).to.be.false; + expect(await helper.applyBurst(12345, 2.0)).to.equal(false); + expect(writeStub.called).to.equal(false); }); it('should write burstUs equal to quotaUs when supported and uncapped', async () => { @@ -320,10 +290,8 @@ describe('cpuBurstHelper tests', () => { sinon.stub(os, 'cpus').returns(new Array(32).fill({ model: 'stub', speed: 2400 })); const accessStub = sinon.stub(fs.promises, 'access').resolves(); const readStub = sinon.stub(fs.promises, 'readFile'); - readStub.withArgs('/proc/version').resolves('Linux version 6.1.0-generic'); readStub.withArgs('/proc/12345/cgroup').resolves('0::/system.slice/docker-x.scope\n'); const writeStub = sinon.stub(fs.promises, 'writeFile').resolves(); - cpuBurstHelper.resetBurstSupportCache(); const result = await cpuBurstHelper.applyBurst(12345, 2.0, 'test-app'); expect(result).to.be.true; diff --git a/tests/unit/dockerService.test.js b/tests/unit/dockerService.test.js index f2788e048c..0c819fd768 100644 --- a/tests/unit/dockerService.test.js +++ b/tests/unit/dockerService.test.js @@ -2336,23 +2336,13 @@ describe('dockerService tests', () => { ...baseNodeApp, containerPorts: [], ports: [], version: 3, }; const created = () => dockerStub.firstCall.args[0]; - const driverIs = (CgroupDriver) => sinon.stub(Dockerode.prototype, 'info').resolves({ CgroupDriver, DriverStatus: [] }); - - it('puts the container in its app\'s slice under the systemd cgroup driver', async () => { - driverIs('systemd'); + it('puts the container in its app\'s slice', async () => { + sinon.stub(Dockerode.prototype, 'info').resolves({ CgroupDriver: 'systemd', DriverStatus: [] }); await dockerService.appDockerCreate(nodeApp, 'my-app', true); expect(created().HostConfig.CgroupParent).to.equal('flux-my\\x2dapp.slice'); }); - - it('leaves the parent to docker under any other cgroup driver', async () => { - driverIs('cgroupfs'); - - await dockerService.appDockerCreate(nodeApp, 'my-app', true); - - expect(created().HostConfig).to.not.have.property('CgroupParent'); - }); }); }); describe('loadImage tests', () => { diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 06e30ac7de..72b9730f8f 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -1125,6 +1125,62 @@ describe('fluxNetworkHelper tests', () => { }); }); + describe('checkCgroupsAllowed tests', () => { + const { CGROUP_FLOOR } = fluxNetworkHelper.StickyDosOwner; + let root; + + // A cgroup filesystem holding the given files. + async function cgroupRoot(files) { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cgroup-')); + await Promise.all(files.map(async (file) => { + await fs.mkdir(path.dirname(path.join(root, file)), { recursive: true }); + await fs.writeFile(path.join(root, file), ''); + })); + return root; + } + const everything = ['cgroup.controllers', 'init.scope/cgroup.kill']; + + afterEach(async () => { + fluxNetworkHelper.clearStickyDos(CGROUP_FLOOR); + fluxNetworkHelper.setDosStateValue(0); + fluxNetworkHelper.setDosMessage(null); + await fs.rm(root, { recursive: true, force: true }); + }); + + it('allows cgroup v2 with cgroup.kill under Docker\'s systemd driver', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot(everything))).to.equal(true); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + }); + + it('takes a node without cgroup v2 out of service, and says so', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot([]))).to.equal(false); + const reported = fluxNetworkHelper.getDOSState().data; + expect(reported.dosMessage).to.include('missing: cgroup v2'); + expect(reported.dosMessage).to.include('restart FluxOS'); + expect(reported.dosState).to.equal(100); + }); + + it('takes a node whose kernel cannot kill a cgroup out of service', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot(['cgroup.controllers']))).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup.kill (Linux 5.14 or later)'); + }); + + it('takes a node whose Docker uses another cgroup driver out of service, and names it', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('cgroupfs', await cgroupRoot(everything))).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: Docker\'s systemd cgroup driver (found cgroupfs)'); + }); + + it('gives no verdict on a driver Docker did not report', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed(null, await cgroupRoot(everything))).to.equal(true); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + }); + + it('still refuses the kernel when Docker did not report its driver', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed(null, await cgroupRoot([]))).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup v2'); + }); + }); + describe('checkDockerVersionAllowed tests', () => { // minimumDockerAllowedVersion = '28.1.1' const { DOCKER_FLOOR, RESIDENTIAL_DOS } = fluxNetworkHelper.StickyDosOwner; From 6e1e468625f10bbb5fd8cac9d229c4a3235abe8d Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 19:47:20 +0100 Subject: [PATCH 232/369] test(harness): 2507, a node without Docker's systemd cgroup driver holds itself out of service Co-Authored-By: Claude Opus 5.5 --- ...roup-driver-holds-itself-out-of-service.js | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js diff --git a/test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js b/test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js new file mode 100644 index 0000000000..34dbe1aba4 --- /dev/null +++ b/test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js @@ -0,0 +1,65 @@ +// A node whose cgroups are not the ones FluxOS runs apps in holds itself out of +// service and says why. Harness nodes run cgroup v2, with cgroup.kill, under Docker's +// systemd cgroup driver, so the node under test has its Docker switched to the +// cgroupfs driver and FluxOS restarted on it; the node beside it keeps the systemd +// driver: the canary that the check passes a node that meets it. The fleet boots +// first, since a node out of service refuses the login the harness boots a fleet +// through. +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, restartDockerd, restartFluxos } from '../framework/container.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const OFF_DRIVER = 0; +const ON_DRIVER = 1; + +const cgroupDriver = async (client) => (await execInContainer(client.container, 'docker info -f {{.CgroupDriver}}')).stdout.trim(); + +describe('2507 a node without Docker\'s systemd cgroup driver holds itself out of service', function suite() { + this.timeout(600000); + + let env; + dumpLogsOnFailure(() => env); + + before(async function hook() { + env = await createTestEnv({ hookCtx: this, nodes: 2, tickerAutostart: false }); + await Promise.all(env.clients.map((client) => waitForBootSettled(client, 240000))); + const node = env.clients[OFF_DRIVER]; + // The driver set in the node's own daemon.json, merged over what the entrypoint wrote. + const switched = await execInContainer(node.container, `node -e ' + const fs = require("fs"); + const target = "/etc/docker/daemon.json"; + const config = fs.existsSync(target) ? JSON.parse(fs.readFileSync(target, "utf8")) : {}; + config["exec-opts"] = ["native.cgroupdriver=cgroupfs"]; + fs.writeFileSync(target, JSON.stringify(config, null, 2)); + '`); + expect(switched.exitCode, switched.stderr).to.equal(0); + await restartDockerd(node.container); + expect(await cgroupDriver(node), 'fixture: Docker did not take the cgroupfs driver').to.equal('cgroupfs'); + await restartFluxos(node.container); + }); + + after(async () => { + await env?.teardown(); + }); + + it('holds the node at DOS 100, naming the driver it found, the one required, and what to do', async () => { + let state; + await waitFor(async () => { + state = (await env.clients[OFF_DRIVER].getDOSState())?.data; + return state?.dosState === 100; + }, { timeout: 90000, label: 'the node without the systemd driver to hold itself out of service' }); + expect(state.dosMessage).to.include('Cgroup Error'); + expect(state.dosMessage).to.include('Docker\'s systemd cgroup driver (found cgroupfs)'); + expect(state.dosMessage).to.include('restart FluxOS'); + }); + + it('does not hold the node whose Docker uses the systemd driver', async () => { + expect(await cgroupDriver(env.clients[ON_DRIVER]), 'fixture: the canary\'s Docker').to.equal('systemd'); + const state = (await env.clients[ON_DRIVER].getDOSState()).data; + expect(state.dosState, state.dosMessage).to.not.equal(100); + expect(String(state.dosMessage ?? '')).to.not.include('Cgroup Error'); + }); +}); From 620d317b822f77b75ef6dce60f79bb87f229c352 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:05:13 +0100 Subject: [PATCH 233/369] fix(apps): the cgroup floor asks for cgroup.kill in FluxOS's own cgroup Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxNetworkHelper.js | 16 ++++++++++------ tests/unit/fluxNetworkHelper.test.js | 20 +++++++++++++++----- 2 files changed, 25 insertions(+), 11 deletions(-) diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 3e3cc40c17..923da2d058 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -1469,19 +1469,23 @@ function checkDockerVersionAllowed(dockerVersion) { * host stall makes all rely on the three. * * Asked once, when Docker first answers at startup. The kernel's answers are - * read from the cgroup filesystem; init.scope is the cgroup systemd always - * creates, and cgroup.kill exists in every cgroup but the root. A driver that - * could not be read gives no verdict on the driver. + * read from the cgroup filesystem: cgroup.kill is asked of FluxOS's own cgroup, + * which /proc/self/cgroup names and which is never the root, the one cgroup + * without the file. A driver that could not be read gives no verdict on the + * driver. * @param {?string} cgroupDriver The cgroup driver Docker reports, or null when * it could not be read. - * @param {string} [cgroupRoot] Where the cgroup filesystem is mounted. + * @param {{cgroupRoot?: string, selfCgroupFile?: string}} [paths] Where the + * cgroup filesystem is mounted, and the file naming this process's cgroup. * @returns {Promise} True unless a requirement is known to be unmet. */ -async function checkCgroupsAllowed(cgroupDriver, cgroupRoot = '/sys/fs/cgroup') { +async function checkCgroupsAllowed(cgroupDriver, { cgroupRoot = '/sys/fs/cgroup', selfCgroupFile = '/proc/self/cgroup' } = {}) { const exists = (file) => fs.access(path.join(cgroupRoot, file)).then(() => true, () => false); + // cgroup v2 lists one line, 0:: + const ownCgroup = (await fs.readFile(selfCgroupFile, 'utf8').catch(() => '')).match(/^0::(\/.*)$/m)?.[1]; const unmet = []; if (!await exists('cgroup.controllers')) unmet.push('cgroup v2'); - else if (!await exists('init.scope/cgroup.kill')) unmet.push('cgroup.kill (Linux 5.14 or later)'); + else if (!ownCgroup || !await exists(path.join(ownCgroup, 'cgroup.kill'))) unmet.push('cgroup.kill (Linux 5.14 or later)'); if (cgroupDriver === null) { log.error('checkCgroupsAllowed - Docker did not report its cgroup driver, no verdict on it'); } else if (cgroupDriver !== 'systemd') { diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 72b9730f8f..2478ad4375 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -1129,16 +1129,20 @@ describe('fluxNetworkHelper tests', () => { const { CGROUP_FLOOR } = fluxNetworkHelper.StickyDosOwner; let root; - // A cgroup filesystem holding the given files. + // A cgroup filesystem holding the given files, with FluxOS in + // system.slice/fluxos.service. async function cgroupRoot(files) { root = await fs.mkdtemp(path.join(os.tmpdir(), 'cgroup-')); + const cgroupFs = path.join(root, 'cgroup'); await Promise.all(files.map(async (file) => { - await fs.mkdir(path.dirname(path.join(root, file)), { recursive: true }); - await fs.writeFile(path.join(root, file), ''); + await fs.mkdir(path.dirname(path.join(cgroupFs, file)), { recursive: true }); + await fs.writeFile(path.join(cgroupFs, file), ''); })); - return root; + const selfCgroupFile = path.join(root, 'self-cgroup'); + await fs.writeFile(selfCgroupFile, '0::/system.slice/fluxos.service\n'); + return { cgroupRoot: cgroupFs, selfCgroupFile }; } - const everything = ['cgroup.controllers', 'init.scope/cgroup.kill']; + const everything = ['cgroup.controllers', 'system.slice/fluxos.service/cgroup.kill']; afterEach(async () => { fluxNetworkHelper.clearStickyDos(CGROUP_FLOOR); @@ -1160,6 +1164,12 @@ describe('fluxNetworkHelper tests', () => { expect(reported.dosState).to.equal(100); }); + it('asks cgroup.kill of FluxOS\'s own cgroup, not of another', async () => { + const paths = await cgroupRoot(['cgroup.controllers', 'init.scope/cgroup.kill']); + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', paths)).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup.kill (Linux 5.14 or later)'); + }); + it('takes a node whose kernel cannot kill a cgroup out of service', async () => { expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot(['cgroup.controllers']))).to.equal(false); expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup.kill (Linux 5.14 or later)'); From 033e2463e85654ef36656f39a8ba1b29eaf7a972 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:10:24 +0100 Subject: [PATCH 234/369] test(harness): every node runs systemd as PID 1, with dockerd and FluxOS as units under Docker's systemd cgroup driver A legacy node the multitool installed runs pm2 as the operator's account, as pm2's own startup unit does. The node's network is built by a unit between dockerd and FluxOS, and the units' output reaches the container stream through a FIFO the entrypoint reads. Co-Authored-By: Claude Opus 5.5 --- test-infra/entrypoint.sh | 334 ++++++------------ test-infra/node-network.sh | 31 ++ test-infra/runner/framework/container.js | 68 ++-- test-infra/runner/framework/log-on-failure.js | 4 +- .../runner/framework/systemd-control.js | 16 +- test-infra/runner/framework/test-env.js | 42 +-- test-infra/runner/run-parallel.sh | 2 +- ...02-policy-reaches-a-node-from-its-peers.js | 6 +- ...02-mount-safety-on-an-unprivileged-node.js | 8 +- ...eturning-node-waits-paused-for-its-role.js | 6 +- .../tests/2401-legacy-maintenance-access.js | 3 +- ...that-stops-answering-takes-nothing-down.js | 10 +- ...lled-host-takes-the-node-out-of-service.js | 1 - .../tests/53-terminal-exec-crash-safety.js | 15 +- test-infra/systemd/dockerd.service | 12 +- test-infra/systemd/flux-node-network.service | 19 + test-infra/systemd/flux-pm2-logs.service | 19 + test-infra/systemd/fluxos.service | 14 +- test-infra/systemd/pm2-fluxuser.service | 25 ++ test-infra/systemd/syncthing-forward.service | 5 +- 20 files changed, 311 insertions(+), 329 deletions(-) create mode 100755 test-infra/node-network.sh create mode 100644 test-infra/systemd/flux-node-network.service create mode 100644 test-infra/systemd/flux-pm2-logs.service create mode 100644 test-infra/systemd/pm2-fluxuser.service diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index b532a90392..16e5568fc4 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -194,7 +194,7 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then fi # Syncthing listens on apiport+2 in production. The availability checker tests -# that port. In systemd mode the stub forward runs as a unit instead +# that port. Against the stub, the forward to it runs as a unit # (syncthing-forward.service), so systemd supervises it like everything else. SYNCTHING_LISTEN_PORT=$((${FLUX_API_PORT:-16127} + 2)) if [ "$FLUX_SYNCTHING_MODE" = "binary" ]; then @@ -225,8 +225,6 @@ if [ "$FLUX_SYNCTHING_MODE" = "binary" ]; then # the flags a real Arcane node is supervised with, read off a live one /flux/test-infra/start-syncthing.sh fi -elif [ -n "$FLUX_SYNCTHING_HOST" ] && [ "$FLUX_SYSTEMD_MODE" != "true" ]; then - socat TCP-LISTEN:${SYNCTHING_LISTEN_PORT},fork,reuseaddr TCP:${FLUX_SYNCTHING_HOST}:${FLUX_SYNCTHING_PORT:-8384} & fi # Trust test registry CA for dockerd (Node.js uses NODE_EXTRA_CA_CERTS directly). @@ -245,234 +243,122 @@ if [ -n "$FLUX_BOOT_ID" ]; then echo "$FLUX_BOOT_ID" > /tmp/flux-boot-id fi -# ── systemd mode (opt-in) ────────────────────────────────────────────────── -# The node runs a real systemd as PID 1: dockerd and fluxos become units, as -# they are on a production host, and anything FluxOS manages through systemctl -# behaves as it does there. Everything below this block is the default path -# and is unreachable in this mode; the fault-injection lever that lives there -# (/tmp/fluxos.pid — restartFluxos) does not exist under systemd, and -# framework/systemd-control.js holds the equivalents. -if [ "$FLUX_SYSTEMD_MODE" = "true" ]; then - - # The Ubuntu base image's policy-rc.d refuses every service start a package - # install asks for. A host has none, so a package installed here starts its - # service as it does on a node. - rm -f /usr/sbin/policy-rc.d - - # Container env does not cross into systemd services (the manager - # environment arrives empty), so dump it for the - # units' EnvironmentFile. node writes C-style-quoted values, which keeps - # NODE_CONFIG's embedded JSON quoting intact. - node -e ' - const fs = require("fs"); - const skip = new Set(["PATH", "HOSTNAME", "HOME", "PWD", "OLDPWD", "SHLVL", "TERM", "SHELL", "_", "DEBIAN_FRONTEND", "LS_COLORS"]); - const lines = Object.entries(process.env) - .filter(([k]) => !skip.has(k)) - .map(([k, v]) => `${k}=${JSON.stringify(v)}`); - fs.writeFileSync("/etc/fluxos-harness.env", lines.join("\n") + "\n"); - ' - - cp /flux/test-infra/systemd/*.service /etc/systemd/system/ - mkdir -p /etc/systemd/system/multi-user.target.wants - ln -sf /etc/systemd/system/dockerd.service /etc/systemd/system/multi-user.target.wants/dockerd.service +# ── systemd ──────────────────────────────────────────────────────────────── +# The node runs a real systemd as PID 1, as a production host does: dockerd and +# FluxOS are units, anything FluxOS manages through systemctl behaves as it does +# there, and Docker places containers through systemd's cgroup driver. + +# The Ubuntu base image's policy-rc.d refuses every service start a package +# install asks for. A host has none, so a package installed here starts its +# service as it does on a node. +rm -f /usr/sbin/policy-rc.d + +# Container env does not cross into systemd services (the manager +# environment arrives empty), so dump it for the +# units' EnvironmentFile. node writes C-style-quoted values, which keeps +# NODE_CONFIG's embedded JSON quoting intact. +node -e ' + const fs = require("fs"); + const skip = new Set(["PATH", "HOSTNAME", "HOME", "PWD", "OLDPWD", "SHLVL", "TERM", "SHELL", "_", "DEBIAN_FRONTEND", "LS_COLORS"]); + const lines = Object.entries(process.env) + .filter(([k]) => !skip.has(k)) + .map(([k, v]) => `${k}=${JSON.stringify(v)}`); + fs.writeFileSync("/etc/fluxos-harness.env", lines.join("\n") + "\n"); +' + +cp /flux/test-infra/systemd/*.service /etc/systemd/system/ +mkdir -p /etc/systemd/system/multi-user.target.wants +for unit in dockerd flux-node-network; do + ln -sf "/etc/systemd/system/$unit.service" "/etc/systemd/system/multi-user.target.wants/$unit.service" +done +# A legacy node the multitool installed runs FluxOS under pm2 as the operator's +# account (pm2-fluxuser.service); every other node runs the fluxos unit. +if [ "$FLUX_PM2" = "1" ]; then + for unit in pm2-fluxuser flux-pm2-logs; do + ln -sf "/etc/systemd/system/$unit.service" "/etc/systemd/system/multi-user.target.wants/$unit.service" + done +else ln -sf /etc/systemd/system/fluxos.service /etc/systemd/system/multi-user.target.wants/fluxos.service - # An unprivileged node runs FluxOS as the operator's account, as an operator's own - # install does; systemd gives the service that account's HOME, from which a legacy - # node derives its flux directory. The apps folder is the account's, and nothing - # else changes hands, as in the default mode below. - if [ -n "$FLUX_FLUXOS_USER" ]; then - mkdir -p /etc/systemd/system/fluxos.service.d - printf '[Service]\nUser=%s\n' "$FLUX_FLUXOS_USER" > /etc/systemd/system/fluxos.service.d/user.conf - chown "$(id -u "$FLUX_FLUXOS_USER"):$(id -g "$FLUX_FLUXOS_USER")" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" - fi - if [ -n "$FLUX_SYNCTHING_HOST" ]; then - ln -sf /etc/systemd/system/syncthing-forward.service /etc/systemd/system/multi-user.target.wants/syncthing-forward.service - fi - - # Container hygiene: kernel modules cannot be loaded here, and a tmpfs - # over /tmp would shadow the harness's /tmp/flux-boot-config bind mount. - # kernel.*, fs.*, vm.* and binfmt_misc are the host's, not the container's, - # and a privileged node can write them: systemd-sysctl and systemd-binfmt - # would apply the image's settings to the shared runner host. - ln -sf /dev/null /etc/systemd/system/systemd-modules-load.service - ln -sf /dev/null /etc/systemd/system/systemd-sysctl.service - ln -sf /dev/null /etc/systemd/system/systemd-binfmt.service - ln -sf /dev/null /etc/systemd/system/tmp.mount - - # docker-ce's packaged containerd.service would boot under systemd and - # dockerd then prefers it — with its snapshotter on the overlayfs root - # (EINVAL on overlay-on-overlay). Masked, dockerd spawns its own - # containerd under the data-root, exactly like the default mode. - ln -sf /dev/null /etc/systemd/system/containerd.service +fi - # docker-ce's own docker.service and docker.socket start a second dockerd on - # the same daemon.json. It takes the volume store's lock, then waits for the - # masked containerd, and dockerd.service can never open the store. Masked, - # dockerd.service is the node's only daemon. - ln -sf /dev/null /etc/systemd/system/docker.service - ln -sf /dev/null /etc/systemd/system/docker.socket +# The node's output: dockerd's, FluxOS's and the network setup's lines go to the +# container's own stream, where the harness reads them, through this FIFO. Its +# reader is started here so it holds the container's stdout; opened read-write, it +# never sees an end of file when a writer stops. +mkdir -p /var/lib/flux-harness +rm -f /var/lib/flux-harness/stream +mkfifo /var/lib/flux-harness/stream +cat 0<>/var/lib/flux-harness/stream & +# An unprivileged node runs FluxOS as the operator's account, as an operator's own +# install does; systemd gives the service that account's HOME, from which a legacy +# node derives its flux directory. The apps folder is the account's, and nothing +# else changes hands. +if [ -n "$FLUX_FLUXOS_USER" ]; then + mkdir -p /etc/systemd/system/fluxos.service.d + printf '[Service]\nUser=%s\n' "$FLUX_FLUXOS_USER" > /etc/systemd/system/fluxos.service.d/user.conf + chown "$(id -u "$FLUX_FLUXOS_USER"):$(id -g "$FLUX_FLUXOS_USER")" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" +fi +if [ -n "$FLUX_SYNCTHING_HOST" ]; then + ln -sf /etc/systemd/system/syncthing-forward.service /etc/systemd/system/multi-user.target.wants/syncthing-forward.service +fi - # The default mode's data-root, in the file the dockerd unit reads its - # configuration from. - mkdir -p /etc/docker - cat > /etc/docker/daemon.json < /etc/docker/daemon.json <{}); w.close();' 2>/dev/null; then - echo "[entrypoint] FATAL: cannot allocate an inotify instance — systemd will exit 255." >&2 - echo "[entrypoint] fs.inotify.max_user_instances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null) is a HOST-WIDE per-uid pool" >&2 - echo "[entrypoint] shared by every container on this box. Raise it on the HOST, not here." >&2 - exit 3 - fi - echo "[entrypoint] setup complete, handing off to systemd as pid 1" - # journal-or-kmsg, not the default: systemd's own messages go to the journal once - # journald exists, and to the KERNEL RING BUFFER before it does — which is the exact - # window a boot failure happens in. The default target leaves that window writing to - # /dev/console, and a container without a TTY has none, so a systemd that dies before - # journald is completely mute. The fallback only fires when the journal is - # unavailable, so a healthy boot writes nothing to kmsg. - exec /lib/systemd/systemd --log-target=journal-or-kmsg -fi - -# Start dockerd under a tiny watchdog so it is respawned if it exits. Production -# nodes run dockerd under systemd (which restarts it); this mirrors that and lets -# tests bounce dockerd (kill it) to exercise the reconciler's reconnect/orphan -# recovery without bricking the node. node app.js stays PID 1 (via exec below). -rm -f /var/run/docker.pid -( - set +e - while true; do - rm -f /var/run/docker.pid - dockerd --data-root /mnt/appdata/docker - echo "dockerd exited (rc=$?), respawning in 1s" >&2 - sleep 1 - done -) & - -TIMEOUT=30 -ELAPSED=0 -until docker info > /dev/null 2>&1; do - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then - echo "ERROR: dockerd failed to start within ${TIMEOUT}s" >&2 - exit 1 - fi - sleep 1 - ELAPSED=$((ELAPSED + 1)) -done -echo "dockerd is ready (took ${ELAPSED}s)" - -# A named network shape (test-infra/network-shapes.sh), declared per node by the -# suite, built before FluxOS starts because a node reads its network at boot. -# Built after dockerd is up, as a VPN comes up after docker on a host: dockerd -# refuses to start when every private range it could give its bridge overlaps a -# route, and the def1 split routes overlap them all. Not swallowed, for the -# reason given for the default route: a shape that failed to build would leave a -# plain static node answering for it. -if [ -n "${FLUX_E2E_NETWORK_SHAPE:-}" ]; then - if ! /flux/test-infra/network-shapes.sh "$FLUX_E2E_NETWORK_SHAPE"; then - echo "ERROR: could not build network shape $FLUX_E2E_NETWORK_SHAPE" >&2 - exit 1 - fi -fi - -# A global IPv6 address, declared per node by the suite (createTestEnv globalIpv6). It sits on -# a device of its own with no IPv6 route beyond it, so the node holds a routable IPv6 address -# as a dual-stack host does while every IPv6 connection fails at once (ENETUNREACH): an IPv6 -# that is configured and does not route. Independent of the network shape, and built before -# FluxOS starts for the same reason. -if [ -n "${FLUX_E2E_GLOBAL_IPV6:-}" ]; then - if ! { ip link add flux6 type dummy \ - && sysctl -qw net.ipv6.conf.flux6.disable_ipv6=0 \ - && ip link set flux6 up \ - && ip -6 addr add "$FLUX_E2E_GLOBAL_IPV6/64" dev flux6 nodad; }; then - echo "ERROR: could not add the global IPv6 address $FLUX_E2E_GLOBAL_IPV6" >&2 - exit 1 - fi -fi - -# WHO FLUXOS RUNS AS. Root unless the fleet names an account, which is the Arcane -# node; named, it is the unprivileged install - FluxOS as the user the operator -# installed it as, with passwordless sudo for everything privileged. -# -# What that account owns is decided here rather than in the image, because it is -# per-node: the tree FluxOS writes into, its logs, and the apps folder, which on an -# unprivileged node belongs to the operator. Nothing else changes hands - dockerd, -# syncthing and every app volume stay root's, and that is what makes the node's own -# reads refusable, exactly as a container's mount point is on a real node. -# -# setpriv rather than sudo: this node's environment IS its configuration, and sudo -# rebuilds the environment it passes on. HOME travels with the account because a -# legacy node derives its flux directory from it. -if [ -n "$FLUX_FLUXOS_USER" ]; then - FLUXOS_UID="$(id -u "$FLUX_FLUXOS_USER")" - FLUXOS_GID="$(id -g "$FLUX_FLUXOS_USER")" - HOME="$(getent passwd "$FLUX_FLUXOS_USER" | cut -d: -f6)" - export HOME - # The apps folder only. The install itself already belongs to this account, from - # the image - a recursive chown of it here would copy the whole tree up into the - # overlay and outlast the node's readiness window. - chown "$FLUXOS_UID:$FLUXOS_GID" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" - set -- setpriv --reuid="$FLUXOS_UID" --regid="$FLUXOS_GID" --init-groups "$@" -fi - -# Run FluxOS (CMD ["node","app.js"]) under a respawn watchdog instead of exec'ing it -# as PID 1. This mirrors the dockerd watchdog above and production's systemd: the -# entrypoint shell stays PID 1 and node runs as a child, so a test can kill+respawn -# the FluxOS process (restartFluxos) WITHOUT restarting the container or the inner -# dockerd - the app containers keep running, exactly like `systemctl restart fluxos`. -# The child PID is written to /tmp/fluxos.pid so a test kills only the node process, -# never PID 1. A SIGTERM/SIGINT (docker stop at teardown) stops the child and exits. +# The handoff marker. Everything above runs as PID 1's shell, whose output IS the +# container's stdout; after it, the units' lines reach that stdout only through the +# stream FIFO, and systemd's own output does not reach it at all. +# So a container that died with this line in `docker logs` failed under systemd (read +# its journal), and one that died WITHOUT it failed in the setup above, where the +# failing command's own stderr is the diagnosis. +# systemd's FIRST act is to create an inotify instance to watch cgroups, and +# fs.inotify.max_user_instances is a PER-UID, HOST-WIDE pool that every container +# on the box draws from as root. Exhaust it and systemd cannot allocate its manager +# object and PID 1 exits 255 — having written the reason to /dev/console, which a +# container without a TTY does not have. That is total silence: no docker logs, no +# journal (journald never started), nothing. # -# While /tmp/fluxos.hold exists the loop does not respawn: the node stays down -# after FluxOS exits, as a machine does between a shutdown and its next boot, -# and comes back once a test removes the file. -# A legacy node the multitool installed: the pm2 daemon starts FluxOS through -# start.sh, which runs npm start - npm install, then FluxOS - from the account's -# home, with the flags the multitool gives it. The kill timeout is the fleet's -# to choose; unset, pm2 uses its own default. There is no registry here, so npm -# answers its install from the image's own. The entrypoint stays in the -# foreground streaming pm2's logs, so FluxOS's output is the container's, and a -# test stops and starts FluxOS through pm2 as an operator or the OS does. -if [ "$FLUX_PM2" = "1" ]; then - export NPM_CONFIG_OFFLINE=true NPM_CONFIG_AUDIT=false NPM_CONFIG_FUND=false NPM_CONFIG_UPDATE_NOTIFIER=false - cd "$HOME" || exit 1 - pm2 start "$HOME/zelflux/start.sh" --name flux \ - --max-memory-restart 1500M --restart-delay 30000 --max-restarts 40 --time \ - ${FLUX_PM2_KILL_TIMEOUT_MS:+--kill-timeout "$FLUX_PM2_KILL_TIMEOUT_MS"} >/dev/null - trap 'pm2 kill >/dev/null 2>&1; exit 0' TERM INT - pm2 logs --raw --lines 0 & - wait $! - exit 0 +# So ask the question here, where the answer can be printed. node is in this image; +# fs.watch allocates exactly the resource systemd is about to need. +if ! node -e 'const w=require("fs").watch("/tmp",()=>{}); w.close();' 2>/dev/null; then + echo "[entrypoint] FATAL: cannot allocate an inotify instance — systemd will exit 255." >&2 + echo "[entrypoint] fs.inotify.max_user_instances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null) is a HOST-WIDE per-uid pool" >&2 + echo "[entrypoint] shared by every container on this box. Raise it on the HOST, not here." >&2 + exit 3 fi - -set +e -STOPPING=0 -trap 'STOPPING=1; kill -TERM "$(cat /tmp/fluxos.pid 2>/dev/null)" 2>/dev/null' TERM INT -while [ "$STOPPING" = "0" ]; do - "$@" & - FLUXOS_PID=$! - echo "$FLUXOS_PID" > /tmp/fluxos.pid - wait "$FLUXOS_PID" - [ "$STOPPING" = "1" ] && break - while [ -f /tmp/fluxos.hold ] && [ "$STOPPING" = "0" ]; do sleep 1; done - [ "$STOPPING" = "1" ] && break - echo "fluxos (node app.js) exited, respawning in 1s" >&2 - sleep 1 -done +echo "[entrypoint] setup complete, handing off to systemd as pid 1" +# journal-or-kmsg, not the default: systemd's own messages go to the journal once +# journald exists, and to the KERNEL RING BUFFER before it does — which is the exact +# window a boot failure happens in. The default target leaves that window writing to +# /dev/console, and a container without a TTY has none, so a systemd that dies before +# journald is completely mute. The fallback only fires when the journal is +# unavailable, so a healthy boot writes nothing to kmsg. +exec /lib/systemd/systemd --log-target=journal-or-kmsg diff --git a/test-infra/node-network.sh b/test-infra/node-network.sh new file mode 100755 index 0000000000..2b8d124b87 --- /dev/null +++ b/test-infra/node-network.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# The node's network as its suite declares it, built once dockerd is up and before +# FluxOS starts (flux-node-network.service), because a node reads its network at boot. +# +# A named network shape (test-infra/network-shapes.sh) is built after dockerd is up, +# as a VPN comes up after docker on a host: dockerd refuses to start when every +# private range it could give its bridge overlaps a route, and the def1 split routes +# overlap them all. Not swallowed: a shape that failed to build would leave a plain +# static node answering for it. +set -e +if [ -n "${FLUX_E2E_NETWORK_SHAPE:-}" ]; then + if ! /flux/test-infra/network-shapes.sh "$FLUX_E2E_NETWORK_SHAPE"; then + echo "ERROR: could not build network shape $FLUX_E2E_NETWORK_SHAPE" >&2 + exit 1 + fi +fi + +# A global IPv6 address, declared per node by the suite (createTestEnv globalIpv6). It +# sits on a device of its own with no IPv6 route beyond it, so the node holds a +# routable IPv6 address as a dual-stack host does while every IPv6 connection fails at +# once (ENETUNREACH): an IPv6 that is configured and does not route. Independent of the +# network shape. +if [ -n "${FLUX_E2E_GLOBAL_IPV6:-}" ]; then + if ! { ip link add flux6 type dummy \ + && sysctl -qw net.ipv6.conf.flux6.disable_ipv6=0 \ + && ip link set flux6 up \ + && ip -6 addr add "$FLUX_E2E_GLOBAL_IPV6/64" dev flux6 nodad; }; then + echo "ERROR: could not add the global IPv6 address $FLUX_E2E_GLOBAL_IPV6" >&2 + exit 1 + fi +fi diff --git a/test-infra/runner/framework/container.js b/test-infra/runner/framework/container.js index c809013f43..b5e7e7bae1 100644 --- a/test-infra/runner/framework/container.js +++ b/test-infra/runner/framework/container.js @@ -228,7 +228,7 @@ export async function getAppContainerExitCode(container, appName, componentName) /** * Bounce the inner dockerd under a running FluxOS (the dockerd-restart orphan - * case). Kills dockerd; the in-image watchdog respawns it. Without --live-restore + * case). Kills dockerd; systemd restarts it (dockerd.service). Without --live-restore * this stops dockerd's containers, leaving them 'exited' for the reconnect sweep * to recover. Confirms dockerd actually went DOWN and came back UP, so the caller * can't observe a false "already ready". @@ -251,19 +251,38 @@ export async function restartDockerd(container, { readyTimeoutMs = 40000, interv throw new Error(`restartDockerd: dockerd did not cycle down and back up within ${readyTimeoutMs}ms`); } +// The node's FluxOS runs as the fluxos unit (test-infra/systemd/fluxos.service), which +// systemd restarts whenever its process exits. A held node is one whose unit carries a +// runtime drop-in turning the restart off: its FluxOS stays down after it exits, as a +// machine does between a shutdown and its next boot, until releaseFluxos. +const FLUXOS_HOLD = '/run/systemd/system/fluxos.service.d/hold.conf'; + +/** + * The pid of the node's FluxOS process, or null while it is not running. + * @returns {Promise} + */ +export async function fluxosPid(container) { + const pid = Number((await execInContainer(container, 'systemctl show -p MainPID --value fluxos')).stdout.trim()); + return pid > 0 ? pid : null; +} + +async function holdFluxos(container) { + const r = await execInContainer(container, + `mkdir -p "$(dirname ${FLUXOS_HOLD})" && printf '[Service]\nRestart=no\n' > ${FLUXOS_HOLD} && systemctl daemon-reload`); + if (r.exitCode !== 0) throw new Error(`holdFluxos: ${r.output}`); +} + /** * Restart the FluxOS process only - the `systemctl restart fluxos` case. Kills just - * the node app.js child (its PID is in /tmp/fluxos.pid, written by the entrypoint - * watchdog, so PID 1 is never touched); the watchdog respawns it. The inner dockerd - * and the running app containers are NOT affected - they keep running while FluxOS's - * in-memory state (e.g. controllerDesired) is wiped. This is distinct from - * restartNode (whole container -> dockerd + containers restart) and restartDockerd - * (dockerd only). Confirms FluxOS went DOWN and came back UP so the caller can't - * observe a false "already ready". + * FluxOS; systemd starts it again. The inner dockerd and the running app containers + * are NOT affected - they keep running while FluxOS's in-memory state (e.g. + * controllerDesired) is wiped. This is distinct from restartNode (whole container -> + * dockerd + containers restart) and restartDockerd (dockerd only). Confirms FluxOS + * went DOWN and came back UP so the caller can't observe a false "already ready". */ export async function restartFluxos(container, { apiPort = 16127, readyTimeoutMs = 120000, interval = 500 } = {}) { - // hard-kill only the node child (state wiped instantly); never PID 1 - await execInContainer(container, 'kill -9 "$(cat /tmp/fluxos.pid 2>/dev/null)" 2>/dev/null || true'); + // hard-kill only FluxOS (state wiped instantly) + await execInContainer(container, 'systemctl kill --kill-whom=main -s KILL fluxos'); const probe = `curl -sf -o /dev/null http://127.0.0.1:${apiPort}/flux/version`; const start = Date.now(); let sawDown = false; @@ -313,9 +332,10 @@ export async function getContainerImageDigest(container, appName, componentName) export async function shutdownFluxosGracefully(container, { hold = false, stopSyncthingAfter = false, exitTimeoutMs = 120000, interval = 250, } = {}) { - const pid = Number((await execInContainer(container, 'cat /tmp/fluxos.pid')).stdout.trim()); - if (!pid) throw new Error('shutdownFluxosGracefully: no FluxOS pid in /tmp/fluxos.pid'); - await execInContainer(container, `touch /run/nologin${hold ? ' /tmp/fluxos.hold' : ''}`); + const pid = await fluxosPid(container); + if (!pid) throw new Error('shutdownFluxosGracefully: FluxOS is not running'); + if (hold) await holdFluxos(container); + await execInContainer(container, 'touch /run/nologin'); if (stopSyncthingAfter) { // Inside the node, so the daemon stops within a poll of the exit rather than // a docker exec round trip later. @@ -360,7 +380,9 @@ export async function shutdownFluxosGracefully(container, { // Bring back a node held down by shutdownFluxosGracefully({ hold: true }), and // wait for its API to answer. export async function releaseFluxos(container, { apiPort = 16127, readyTimeoutMs = 120000, interval = 500 } = {}) { - await execInContainer(container, 'rm -f /tmp/fluxos.hold'); + const released = await execInContainer(container, + `rm -f ${FLUXOS_HOLD} && systemctl daemon-reload && systemctl reset-failed fluxos; systemctl start fluxos`); + if (released.exitCode !== 0) throw new Error(`releaseFluxos: ${released.output}`); const probe = `curl -sf -o /dev/null http://127.0.0.1:${apiPort}/flux/version`; const start = Date.now(); while (Date.now() - start < readyTimeoutMs) { @@ -418,9 +440,9 @@ export async function configuredIpOf(container) { // shutdown handling at all. `hold` keeps the node down afterwards, as a machine // that lost power stays down; releaseFluxos brings it back. export async function crashFluxos(container, { hold = false, exitTimeoutMs = 30000, interval = 250 } = {}) { - const pid = Number((await execInContainer(container, 'cat /tmp/fluxos.pid')).stdout.trim()); - if (!pid) throw new Error('crashFluxos: no FluxOS pid in /tmp/fluxos.pid'); - if (hold) await execInContainer(container, 'touch /tmp/fluxos.hold'); + const pid = await fluxosPid(container); + if (!pid) throw new Error('crashFluxos: FluxOS is not running'); + if (hold) await holdFluxos(container); await execInContainer(container, `kill -9 ${pid}`); const start = Date.now(); while (Date.now() - start < exitTimeoutMs) { @@ -433,17 +455,21 @@ export async function crashFluxos(container, { hold = false, exitTimeoutMs = 300 throw new Error(`crashFluxos: FluxOS ${pid} still running ${exitTimeoutMs}ms after SIGKILL`); } +// pm2 as a legacy node's operator runs it: as their account, against their pm2 home +// (test-infra/systemd/pm2-fluxuser.service). +const PM2 = 'runuser -u fluxuser -- env PM2_HOME=/home/fluxuser/.pm2 pm2'; + // The kill timeout of FluxOS's pm2 registration on a legacy node whose FluxOS // pm2 runs (createTestEnv pm2Nodes), as the daemon holds it and as `pm2 save` // stored it for the next boot. null where pm2 has none, which is its default. // // @returns {Promise<{live: number|null, saved: number|null, restarts: number}>} export async function pm2Registration(container) { - const jlist = await execInContainer(container, 'pm2 jlist'); + const jlist = await execInContainer(container, `${PM2} jlist`); if (jlist.exitCode !== 0) throw new Error(`pm2Registration: pm2 jlist failed: ${jlist.output}`); const live = JSON.parse(jlist.stdout.trim()).find((p) => p.name === 'flux'); if (!live) throw new Error('pm2Registration: pm2 lists no process named flux'); - const dump = await execInContainer(container, 'cat "$HOME/.pm2/dump.pm2" 2>/dev/null || echo "[]"'); + const dump = await execInContainer(container, 'cat /home/fluxuser/.pm2/dump.pm2 2>/dev/null || echo "[]"'); const saved = JSON.parse(dump.stdout.trim()).find((p) => p.name === 'flux'); return { live: live.pm2_env.kill_timeout ?? null, @@ -464,7 +490,7 @@ export async function shutdownFluxosUnderPm2(container, { stopSyncthingAfter = f await execInContainer(container, 'touch /run/nologin'); try { const started = Date.now(); - const stopped = await execInContainer(container, 'pm2 stop flux'); + const stopped = await execInContainer(container, `${PM2} stop flux`); const stopMs = Date.now() - started; if (stopped.exitCode !== 0) throw new Error(`shutdownFluxosUnderPm2: pm2 stop failed: ${stopped.output}`); if (stopSyncthingAfter) await execInContainer(container, 'pkill -KILL -x syncthing; true'); @@ -477,7 +503,7 @@ export async function shutdownFluxosUnderPm2(container, { stopSyncthingAfter = f // Start FluxOS again through pm2 on a node shut down by shutdownFluxosUnderPm2, // and wait for its API to answer. export async function startFluxosUnderPm2(container, { apiPort = 16127, readyTimeoutMs = 180000, interval = 1000 } = {}) { - const started = await execInContainer(container, 'pm2 start flux'); + const started = await execInContainer(container, `${PM2} start flux`); if (started.exitCode !== 0) throw new Error(`startFluxosUnderPm2: pm2 start failed: ${started.output}`); const probe = `curl -sf -o /dev/null http://127.0.0.1:${apiPort}/flux/version`; const start = Date.now(); diff --git a/test-infra/runner/framework/log-on-failure.js b/test-infra/runner/framework/log-on-failure.js index a418d17aeb..1da92221ed 100644 --- a/test-infra/runner/framework/log-on-failure.js +++ b/test-infra/runner/framework/log-on-failure.js @@ -163,8 +163,8 @@ export function dumpLogsOnFailure(getEnv) { const cgroupsByEnv = await Promise.all(envs.map((env) => cgroupState(env).catch( (err) => `cgroup probe failed: ${err.message}\n`, ))); - // A systemd-mode node's FluxOS writes to its journal, never to the container - // stream the log collectors read; see env.nodeJournals. + // What a node's units write to the journal rather than to the container stream + // the log collectors read; see env.nodeJournals. const journalsByEnv = await Promise.all(envs.map((env) => env.nodeJournals().catch( (err) => [{ index: 0, text: `journal read failed: ${err.message}\n` }], ))); diff --git a/test-infra/runner/framework/systemd-control.js b/test-infra/runner/framework/systemd-control.js index 60bfab9e6f..f95dcbaf25 100644 --- a/test-infra/runner/framework/systemd-control.js +++ b/test-infra/runner/framework/systemd-control.js @@ -1,10 +1,7 @@ -// Node control for systemd-mode nodes (createTestEnv({ systemdMode: true })). -// -// The default entrypoint's levers in container.js (restartFluxos, -// restartDockerd) drive a shell watchdog that does not exist here: under systemd -// FluxOS and dockerd are real units, so the equivalents are systemctl calls — -// which is also how a production host does it, making these the more faithful -// instruments of the two. +// Node control through systemd, which every node runs as PID 1: FluxOS and dockerd +// are units, controlled with systemctl as on a production host. container.js holds +// the faults (a FluxOS killed outright, a dockerd killed); these are the orderly +// stops and starts. // // The unit names mirror production: `fluxos` is the name an Arcane node's unit // carries. @@ -83,8 +80,9 @@ export async function startFluxos(container, { apiPort = 16127, readyTimeoutMs = } /** - * The systemd-mode restartFluxos: FluxOS's in-memory state is wiped while - * dockerd and the app containers keep running. Observably down, then up. + * `systemctl restart fluxos`, as an operator runs it: FluxOS stops in order and + * starts again, its in-memory state wiped while dockerd and the app containers keep + * running. Observably down, then up. */ export async function restartFluxos(container, opts = {}) { await stopFluxos(container, opts); diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 2b0cd0e5a3..5bcfbb20ef 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -368,10 +368,9 @@ class StaticIpContainer extends GenericContainer { return this; } - // Per-container stop signal (the image's STOPSIGNAL stays SIGTERM for the - // default entrypoint). systemd-mode nodes need SIGRTMIN+3: systemd as PID 1 - // treats SIGTERM as a reexec request, so a plain docker stop would sit out - // the full kill timeout on every teardown. + // Per-container stop signal. A node needs SIGRTMIN+3: systemd as PID 1 treats + // SIGTERM as a reexec request, so a plain docker stop would sit out the full kill + // timeout on every teardown. withStopSignal(signal) { this.#stopSignal = signal; return this; @@ -634,9 +633,8 @@ function makeEnvShell(networkName) { return [...byIndex.values()].sort((a, b) => a.index - b.index); }, - // Each systemd-mode node's journal, read from the host copy, so a node whose - // container is already gone - one that never finished booting - still has its - // record. Empty for a default-mode env, whose nodes log to the container stream. + // Each node's journal, read from the host copy, so a node whose container is + // already gone - one that never finished booting - still has its record. async nodeJournals() { return Promise.all(nodeConfigs .filter((cfg) => cfg.journalDir && existsSync(cfg.journalDir)) @@ -974,10 +972,6 @@ export async function createTestEnv({ rpcFailures = [], bootContext = 'running', initialHeight = DEFAULT_INITIAL_HEIGHT, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation = {}, locationTable = null, staticIp = true, policy = null, policySeeds = null, awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], - // systemd-mode nodes: the entrypoint execs a real systemd as PID 1, so dockerd - // and fluxos run as units and FluxOS's output goes to the journal. Env-level - // and uniform across nodes. - systemdMode = false, // Firewalled nodes boot with ufw active as their install leaves it - the legacy // installer's rules on a legacy node, the ISO's on an Arcane one - and FluxOS // adds its own on top. true for every node, or a list of node indices. Off @@ -1284,7 +1278,7 @@ export async function createTestEnv({ // mongo starts, i.e. inside the fleet boot, where the waits at risk are the // boot's own. await startInfraDeathWatch(env); - await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, systemdMode, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot); + await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot); return env; } catch (err) { // Boot failed: the env owns everything started so far. The shared teardown @@ -1313,7 +1307,7 @@ function mergeConfigs(base, override) { return result; } -async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], systemdMode = false, firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = []) { +async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = []) { // Everything built here registers onto the env shell as it comes up, so a // boot-phase throw leaves the partial state reachable (see makeEnvShell). const { @@ -1627,7 +1621,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // the rest of the run's record. // Kept for passing suites too: a green suite is exactly where a node that died // and was never noticed would hide, and this record cannot be reconstructed later. - const journalDir = systemdMode ? join(runLogDir(), `journal-${networkName}-${num}`) : null; + const journalDir = join(runLogDir(), `journal-${networkName}-${num}`); if (journalDir) { mkdirSync(journalDir, { recursive: true }); bindMounts.push( @@ -1688,8 +1682,10 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // can READ - syncthing is spawned with sudo whichever account FluxOS holds, so on // an unprivileged node the index can describe files the node itself is refused. if (unprivilegedNodes.includes(i)) nodeEnv.FLUX_FLUXOS_USER = 'fluxuser'; + // pm2 runs as the account the multitool installed under, and FluxOS as pm2 does. if (Object.prototype.hasOwnProperty.call(pm2Nodes, i)) { nodeEnv.FLUX_PM2 = '1'; + nodeEnv.FLUX_FLUXOS_USER = 'fluxuser'; if (pm2Nodes[i] != null) nodeEnv.FLUX_PM2_KILL_TIMEOUT_MS = String(pm2Nodes[i]); } // Legacy only, because it is the only node type that installs anything: @@ -1697,7 +1693,6 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // of syncthing would simply never get it back. if (!aptSeeded && isLegacy) nodeEnv.FLUX_APT_SEEDED = 'false'; if (aptBadSource && isLegacy) nodeEnv.FLUX_APT_BAD_SOURCE = 'true'; - if (systemdMode) nodeEnv.FLUX_SYSTEMD_MODE = 'true'; if (firewall === true || (Array.isArray(firewall) && firewall.includes(i))) nodeEnv.FLUX_FIREWALL = 'true'; // Point the node's config at the base-derived infra IPs. The mounted config // files carry the default 31.200.0 addresses; this is written into the node's @@ -1819,7 +1814,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod .withLogConsumer(logCollector) .withEnvironment(nodeEnv) .withWaitStrategy(nodeReadyWaitStrategy(nodeIp).withStartupTimeout(120000)); - if (systemdMode) builder.withStopSignal('SIGRTMIN+3'); + builder.withStopSignal('SIGRTMIN+3'); if (nodeAppArmor()) builder.withAppArmorProfile(nodeAppArmor()); nodeConfigs.push({ @@ -1971,18 +1966,6 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // Nodes this fixture has cut off from the fleet. They reach nobody and nobody reaches // them, so they peer with nobody and obtain no policy - by the fixture's own doing. const heldOut = new Set(); - // In systemd mode the container's stdout is systemd's console and FluxOS's - // own stream goes to the journal, so the log collectors legitimately receive - // nothing from FluxOS. Answering "no match" would leave every assertion built - // on these silently unfalsifiable, so refuse and name the replacement. - const assertCollectorSeesFluxos = (fn) => { - if (!systemdMode) return; - throw new Error( - `${fn}() cannot see FluxOS logs in systemd mode: its stdout goes to the journal, ` - + 'not the container stream. Read the journal instead — journalGrep(container, ' - + "'fluxos', pattern, { processOnly: true }) from framework/systemd-control.js", - ); - }; // Post-boot methods join the shell here (they close over _buildEnv locals like // deferredBuilders/fluxNodes); identity, registries and teardown live on the @@ -2372,17 +2355,14 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod }, nodeHasLog(index, pattern) { - assertCollectorSeesFluxos('nodeHasLog'); return fluxNodes[index].logCollector.hasLine(pattern); }, nodeLogCount(index, pattern) { - assertCollectorSeesFluxos('nodeLogCount'); return fluxNodes[index].logCollector.countPattern(pattern); }, nodeLogLines(index) { - assertCollectorSeesFluxos('nodeLogLines'); return fluxNodes[index].logCollector.getLines(); }, }); diff --git a/test-infra/runner/run-parallel.sh b/test-infra/runner/run-parallel.sh index b0dc6b947f..2567a8541d 100755 --- a/test-infra/runner/run-parallel.sh +++ b/test-infra/runner/run-parallel.sh @@ -145,7 +145,7 @@ if [ "$ino_now" -lt "$INOTIFY_MIN" ]; then ino_now=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0) if [ "$ino_now" -lt "$INOTIFY_MIN" ]; then abort 94 "###ABORT inotify instance pool too small (max_user_instances=$ino_now < $INOTIFY_MIN) and could not raise it." \ - "Every systemd-mode node needs one and the pool is host-wide, so a gate on this host" \ + "Every node's systemd needs one and the pool is host-wide, so a gate on this host" \ "produces silent exit-255 boot deaths by design." \ "Raise it for the run: sudo sysctl -w fs.inotify.max_user_instances=$INOTIFY_MIN" \ "Or lower the bar deliberately with E2E_INOTIFY_MAX_USER_INSTANCES." diff --git a/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js b/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js index 74e34b6e95..8e6f18b172 100644 --- a/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js +++ b/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js @@ -10,7 +10,7 @@ import { getSubnetConfig } from '../framework/subnet-config.js'; import { waitFor, waitForBootSettled } from '../framework/wait.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; import { buildAppSpec } from '../framework/app-helper.js'; -import { execInContainer } from '../framework/container.js'; +import { execInContainer, fluxosPid } from '../framework/container.js'; // Policy arriving from a PEER rather than from the published source. // @@ -175,12 +175,12 @@ describe('policy reaching a node from its peers', function () { // message, so a failure inside it has nowhere to go but the process - and the node's // answer to that is to exit. Sent from the stub exactly as a real peer would. const client = env.clients[TOLD_NODE]; - const readPid = async () => (await execInContainer(client.container, 'cat /tmp/fluxos.pid')).stdout.trim(); + const readPid = () => fluxosPid(client.container); const pidBefore = await readPid(); // A pid that was never read is equal to itself afterwards, so the comparison below // would hold for a node that had died and come back twice. - expect(pidBefore, 'no FluxOS pid to compare against').to.match(/^[0-9]+$/); + expect(pidBefore, 'no FluxOS pid to compare against').to.be.a('number'); const seqBefore = await heldSeq(TOLD_NODE); // Not text. Measuring it raises before anything is parsed and before any signature is diff --git a/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js b/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js index 1f4d8e9497..01d45ae7d4 100644 --- a/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js +++ b/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js @@ -1,7 +1,7 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { execInContainer, getAppContainerStatus } from '../framework/container.js'; +import { execInContainer, getAppContainerStatus, fluxosPid } from '../framework/container.js'; import { setSyncState, setSynced, setSyncing, getSyncthingState, resetSyncState, injectSyncthingEvent, @@ -48,10 +48,10 @@ async function folderType(nodeIp, folderId) { return node?.folders?.find((f) => f.id === folderId)?.type ?? null; } -// The account the node's own FluxOS process holds, read off the process the -// entrypoint supervises rather than off the container, which stays root. +// The account the node's own FluxOS process holds, read off the process rather +// than off the container, which stays root. async function fluxosAccount(client) { - const r = await execInContainer(client.container, 'sh -c \'ps -o user= -p "$(cat /tmp/fluxos.pid)"\''); + const r = await execInContainer(client.container, `ps -o user= -p ${await fluxosPid(client.container)}`); return r.output.trim(); } diff --git a/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js b/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js index aeb5c992fc..ef8ee17a35 100644 --- a/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js +++ b/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js @@ -5,7 +5,7 @@ import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { execInContainer, isAppContainerRunning, crashFluxos, releaseFluxos, shutdownFluxosGracefully, - redirectOutbound, clearOutboundRedirect, + redirectOutbound, clearOutboundRedirect, fluxosPid, } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; @@ -218,8 +218,8 @@ describe('a returning node waits paused until its role is decided', function () this.timeout(300000); const since = await lastDaemonEventId(client(ARCANE)); const mark = client(ARCANE).getLastEventId(); - const pid = (await sh(ARCANE, 'cat /tmp/fluxos.pid')).stdout.trim(); - expect(pid, 'fixture: no FluxOS pid').to.match(/^\d+$/); + const pid = await fluxosPid(client(ARCANE).container); + expect(pid, 'fixture: no FluxOS pid').to.be.a('number'); // SIGTERM with no system shutdown under way: the service restarting. await sh(ARCANE, `kill -TERM ${pid}`); diff --git a/test-infra/runner/tests/2401-legacy-maintenance-access.js b/test-infra/runner/tests/2401-legacy-maintenance-access.js index 1a33784e8a..8017438049 100644 --- a/test-infra/runner/tests/2401-legacy-maintenance-access.js +++ b/test-infra/runner/tests/2401-legacy-maintenance-access.js @@ -1,4 +1,4 @@ -// Maintenance SSH access on a legacy node, end to end on systemd-mode nodes: the +// Maintenance SSH access on a legacy node, end to end: the // fluxadm account, its passwordless sudo, the configured keys and a dedicated sshd // unit on apiport - 5, reached over the fleet network with a real ssh client. // @@ -327,7 +327,6 @@ describe('2401 legacy node maintenance access', function suite() { nodes: 7, legacyNodes: [LEGACY, NODE_OWNER_SSHD, ...FAULTED], firewall: [LEGACY], - systemdMode: true, tickerAutostart: false, configOverrides: { fluxadm: { sshAuthorizedKeys: [publicKey('current')] } }, }); diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js index 2f554b6550..d1ed60ecc4 100644 --- a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -4,7 +4,7 @@ import { createTestEnv } from '../framework/test-env.js'; import { authenticate } from '../auth.js'; import { appOwnerKey } from '../framework/keys.js'; import { - execInContainer, getAppContainerStatus, crashFluxos, releaseFluxos, + execInContainer, getAppContainerStatus, crashFluxos, releaseFluxos, fluxosPid, } from '../framework/container.js'; import { waitFor, waitForUp, waitForDown, waitForOperatorIntent, waitForBootSettled, assertNoEvent, @@ -54,10 +54,6 @@ function step(what) { console.log(`# 3301 ${new Date().toISOString()} ${what}`); } -async function fluxosPid(client) { - return Number((await execInContainer(client.container, 'cat /tmp/fluxos.pid')).stdout.trim()); -} - describe('a database that stops answering takes nothing down', function () { let env; dumpLogsOnFailure(() => env); @@ -103,7 +99,7 @@ describe('a database that stops answering takes nothing down', function () { before(async function () { this.timeout(HOLD_MS + 120000); step('reading the FluxOS pids'); - pidsBefore = await Promise.all(env.clients.map(fluxosPid)); + pidsBefore = await Promise.all(env.clients.map((c) => fluxosPid(c.container))); anchors = env.clients.map((client) => client.getLastEventId()); step('holding the commands'); await holdCommands(HOLD_MS); @@ -137,7 +133,7 @@ describe('a database that stops answering takes nothing down', function () { }); it('keeps every FluxOS process it had', async function () { - const pidsAfter = await Promise.all(env.clients.map(fluxosPid)); + const pidsAfter = await Promise.all(env.clients.map((c) => fluxosPid(c.container))); expect(pidsAfter, 'a node\'s FluxOS exited during the hold').to.deep.equal(pidsBefore); }); diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index bb33b8f781..06b8120216 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -89,7 +89,6 @@ describe('a stalled host takes the node out of service', function () { hookCtx: this, nodes: 4, tickerAutostart: false, - systemdMode: true, freezableDockerRoot: [0, 1, 2, 3], legacyNodes: [LEGACY_UNPRIVILEGED], unprivilegedNodes: [LEGACY_UNPRIVILEGED], diff --git a/test-infra/runner/tests/53-terminal-exec-crash-safety.js b/test-infra/runner/tests/53-terminal-exec-crash-safety.js index ffc289620a..862435b505 100644 --- a/test-infra/runner/tests/53-terminal-exec-crash-safety.js +++ b/test-infra/runner/tests/53-terminal-exec-crash-safety.js @@ -4,7 +4,7 @@ import { io } from 'socket.io-client'; import { createTestEnv } from '../framework/test-env.js'; import { authenticate } from '../auth.js'; import { appOwnerKey } from '../framework/keys.js'; -import { execInContainer, getAppContainerStatus } from '../framework/container.js'; +import { execInContainer, getAppContainerStatus, fluxosPid } from '../framework/container.js'; import { waitFor } from '../framework/wait.js'; import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -28,17 +28,10 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // // The assertion that matters in both cases is the same: a socket error reaches the // client AND the FluxOS process is the same one it was before (an exit(1) is -// invisible from outside once the watchdog respawns it, so we compare the pid). +// invisible from outside once systemd starts it again, so we compare the pid). const TERMINAL_TIMEOUT_MS = 20000; -// The node app.js child pid, written by the image entrypoint watchdog. It changes -// if and only if FluxOS died and was respawned. -async function fluxosPid(container) { - const { stdout } = await execInContainer(container, 'cat /tmp/fluxos.pid 2>/dev/null || echo ""'); - return stdout.trim(); -} - async function apiAlive(client) { const res = await client.get('/flux/version').catch(() => null); return !!res; @@ -119,7 +112,7 @@ describe('docker terminal fails cleanly and never crashes the node', function () this.timeout(180000); const pidBefore = await fluxosPid(client.container); - expect(pidBefore).to.not.equal(''); + expect(pidBefore).to.not.equal(null); // A real session against the harness stub image, which has no shell and no // passwd: exec the one binary it carries (/bin/pause) as the image-default @@ -179,7 +172,7 @@ describe('docker terminal fails cleanly and never crashes the node', function () }, { timeout: 60000, interval: 2000, label: 'app container stopped' }); const pidBefore = await fluxosPid(client.container); - expect(pidBefore).to.not.equal(''); + expect(pidBefore).to.not.equal(null); const result = await openTerminal(client, zelidauth, identifier); diff --git a/test-infra/systemd/dockerd.service b/test-infra/systemd/dockerd.service index 1786443716..23d258ce1b 100644 --- a/test-infra/systemd/dockerd.service +++ b/test-infra/systemd/dockerd.service @@ -1,15 +1,21 @@ [Unit] -Description=Inner Docker daemon (harness node, systemd mode) +Description=Inner Docker daemon (harness node) After=network.target +# Restarted as often as a suite asks: a test that restarts it repeatedly is not a +# unit failing. +StartLimitIntervalSec=0 [Service] ExecStartPre=/bin/rm -f /var/run/docker.pid -# No flags: /etc/docker/daemon.json (written by the entrypoint's systemd -# branch) owns the configuration — dockerd refuses the same directive from +# No flags: /etc/docker/daemon.json (written by the entrypoint) owns the +# configuration — dockerd refuses the same directive from # both a flag and the file. ExecStart=/usr/bin/dockerd Restart=always RestartSec=1 +# The node's output stream, where the harness reads the unit's lines (entrypoint.sh). +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit Delegate=yes LimitNOFILE=1048576 diff --git a/test-infra/systemd/flux-node-network.service b/test-infra/systemd/flux-node-network.service new file mode 100644 index 0000000000..44ee2ebed3 --- /dev/null +++ b/test-infra/systemd/flux-node-network.service @@ -0,0 +1,19 @@ +[Unit] +Description=Node network shape and global IPv6 (harness node) +# Built once dockerd answers and before FluxOS starts: a node reads its network at +# boot, and dockerd refuses to start under the routes a shape adds. +After=dockerd.service +Requires=dockerd.service +Before=fluxos.service pm2-fluxuser.service + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/etc/fluxos-harness.env +ExecStartPre=/bin/bash -c 'for i in $$(seq 1 30); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1' +ExecStart=/flux/test-infra/node-network.sh +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit + +[Install] +WantedBy=multi-user.target diff --git a/test-infra/systemd/flux-pm2-logs.service b/test-infra/systemd/flux-pm2-logs.service new file mode 100644 index 0000000000..99141371b0 --- /dev/null +++ b/test-infra/systemd/flux-pm2-logs.service @@ -0,0 +1,19 @@ +[Unit] +# pm2 writes FluxOS's output to its own log files; this streams them into the +# node's output, where the harness reads FluxOS's lines. +Description=pm2 log stream (harness node, legacy multitool install) +After=pm2-fluxuser.service +BindsTo=pm2-fluxuser.service + +[Service] +User=fluxuser +EnvironmentFile=/etc/fluxos-harness.env +Environment=PM2_HOME=/home/fluxuser/.pm2 +ExecStart=/usr/bin/env pm2 logs --raw --lines 0 +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit +Restart=always +RestartSec=1 + +[Install] +WantedBy=multi-user.target diff --git a/test-infra/systemd/fluxos.service b/test-infra/systemd/fluxos.service index 1f2907f71f..1d182ec570 100644 --- a/test-infra/systemd/fluxos.service +++ b/test-infra/systemd/fluxos.service @@ -1,8 +1,12 @@ [Unit] # The unit carries the production unit's name. -Description=FluxOS (harness node, systemd mode) -After=dockerd.service +Description=FluxOS (harness node) +After=dockerd.service flux-node-network.service Wants=dockerd.service +Requires=flux-node-network.service +# Restarted as often as a suite asks: a test that restarts it repeatedly is not a +# unit failing. +StartLimitIntervalSec=0 [Service] # User= is what makes systemd set HOME, and a legacy node finds its install at @@ -14,12 +18,14 @@ WorkingDirectory=/flux # environment arrives empty) — the entrypoint dumps it here before # exec'ing systemd. EnvironmentFile=/etc/fluxos-harness.env -# Mirror the legacy entrypoint's gate: FluxOS boots only once the inner -# dockerd answers. +# FluxOS boots only once the inner dockerd answers. ExecStartPre=/bin/bash -c 'for i in $$(seq 1 30); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1' ExecStart=/usr/bin/node app.js Restart=always RestartSec=1 +# The node's output stream, where the harness reads the unit's lines (entrypoint.sh). +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit [Install] WantedBy=multi-user.target diff --git a/test-infra/systemd/pm2-fluxuser.service b/test-infra/systemd/pm2-fluxuser.service new file mode 100644 index 0000000000..11bb87d5e9 --- /dev/null +++ b/test-infra/systemd/pm2-fluxuser.service @@ -0,0 +1,25 @@ +[Unit] +# A legacy node the multitool installed, as `pm2 startup` registers it: pm2 runs as +# the operator's account and starts FluxOS through start.sh, which runs npm start - +# npm install, then FluxOS - from the account's home, with the flags the multitool +# gives it. The kill timeout is the fleet's to choose; unset, pm2 uses its own default. +Description=PM2 process manager +After=dockerd.service flux-node-network.service +Wants=dockerd.service +Requires=flux-node-network.service + +[Service] +Type=forking +User=fluxuser +EnvironmentFile=/etc/fluxos-harness.env +Environment=PM2_HOME=/home/fluxuser/.pm2 +# There is no registry here, so npm answers its install from the image's own. +Environment=NPM_CONFIG_OFFLINE=true NPM_CONFIG_AUDIT=false NPM_CONFIG_FUND=false NPM_CONFIG_UPDATE_NOTIFIER=false +PIDFile=/home/fluxuser/.pm2/pm2.pid +ExecStart=/bin/bash -c 'cd "$$HOME" && exec pm2 start "$$HOME/zelflux/start.sh" --name flux --max-memory-restart 1500M --restart-delay 30000 --max-restarts 40 --time $${FLUX_PM2_KILL_TIMEOUT_MS:+--kill-timeout "$$FLUX_PM2_KILL_TIMEOUT_MS"}' +ExecStop=/usr/bin/env pm2 kill +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit + +[Install] +WantedBy=multi-user.target diff --git a/test-infra/systemd/syncthing-forward.service b/test-infra/systemd/syncthing-forward.service index 0fd517fec1..d487a170ed 100644 --- a/test-infra/systemd/syncthing-forward.service +++ b/test-infra/systemd/syncthing-forward.service @@ -1,11 +1,10 @@ [Unit] -Description=Syncthing stub port forward (apiport+2, harness node, systemd mode) +Description=Syncthing stub port forward (apiport+2, harness node) [Service] EnvironmentFile=/etc/fluxos-harness.env # $$ keeps the shell's own expansion out of systemd's specifier/variable -# expansion; the arithmetic and defaults run in bash, exactly like the -# legacy entrypoint's forward. +# expansion; the arithmetic and defaults run in bash. ExecStart=/bin/bash -c 'exec socat TCP-LISTEN:$$(( $${FLUX_API_PORT:-16127} + 2 )),fork,reuseaddr TCP:$${FLUX_SYNCTHING_HOST}:$${FLUX_SYNCTHING_PORT:-8384}' Restart=always RestartSec=1 From 1a9bae1b1ed32a237026ad12cb49aad8f979f95a Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:13:10 +0100 Subject: [PATCH 235/369] test(harness): a gate raises the host's inotify instance pool to 8192 Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/run-parallel.sh | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/test-infra/runner/run-parallel.sh b/test-infra/runner/run-parallel.sh index 2567a8541d..766587b6a5 100755 --- a/test-infra/runner/run-parallel.sh +++ b/test-infra/runner/run-parallel.sh @@ -129,14 +129,9 @@ fi # - the silent exit-255 class that went unexplained for three gates and survived # six reproduction attempts, because a solo run never exhausts a host-wide pool. # -# The arithmetic is why it was intermittent rather than constant. Measured directly -# once the gauge existed (gate 10): a full gate peaks at 102 concurrent containers -# and 221 inotify instances - 2.17 each, not the 1.27 a quieter sample suggested. -# So the harness has been running at 1.7x the default ceiling of 128, and the wonder -# is not that gates failed but that any passed: containers hold an instance only -# briefly, so a boot survived or died on the exact overlap. 1024 is 4.6x the measured -# peak, which leaves room for wider fleets rather than merely clearing today's number. -INOTIFY_MIN="${E2E_INOTIFY_MAX_USER_INSTANCES:-1024}" +# Every node's systemd holds instances for as long as the node runs, so the pool is +# sized far above any gate's need: an instance costs only a little kernel memory. +INOTIFY_MIN="${E2E_INOTIFY_MAX_USER_INSTANCES:-8192}" require_int E2E_INOTIFY_MAX_USER_INSTANCES "$INOTIFY_MIN" ino_now=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0) require_int fs.inotify.max_user_instances "$ino_now" From 77e9ffa2c05be3cf3760eaaab09a3bcb1458dae6 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:16:46 +0100 Subject: [PATCH 236/369] test(harness): a node's node_modules belongs to the account that owns its install Co-Authored-By: Claude Opus 5.5 --- test-infra/Dockerfile.fluxos | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test-infra/Dockerfile.fluxos b/test-infra/Dockerfile.fluxos index 35a763b76e..742fefb79e 100644 --- a/test-infra/Dockerfile.fluxos +++ b/test-infra/Dockerfile.fluxos @@ -177,8 +177,12 @@ RUN set -eu; \ WORKDIR /flux -COPY package.json ./ -RUN npm install --production \ +# node_modules belongs to the account that owns the install, as it does on an +# operator's node: pm2 starts FluxOS through npm start, which runs npm install as +# that account. Installed as it, so no chown copies the tree into a layer of its own. +RUN chown fluxuser:fluxuser /flux +COPY --chown=fluxuser:fluxuser package.json ./ +RUN runuser -u fluxuser -- npm install --production \ && cp package-lock.json /tmp/image-package-lock.json # pm2, for a legacy node started the way the multitool starts one, at the @@ -203,9 +207,9 @@ RUN PM2_VERSION="$(tr -d '[:space:]' < /tmp/pm2-version)" \ # key it generates at first boot. Carried by the copy rather than a chown of its # own: on an overlay filesystem a recursive chown copies every file it touches up # into the container's writable layer, which on this tree is minutes of boot and -# reads as a node that never came up. node_modules keeps root's ownership - it is -# .dockerignore'd here and installed above, and FluxOS only reads it. Root ignores -# all of it, so the image is unchanged for a node that stays root. +# reads as a node that never came up. node_modules is .dockerignore'd here and +# installed above. Root ignores all of it, so the image is unchanged for a node that +# stays root. COPY --chown=fluxuser:fluxuser . . COPY --chown=fluxuser:fluxuser test-infra/fixtures/userconfig-template.js /flux/config/userconfig.js @@ -214,7 +218,7 @@ COPY --chown=fluxuser:fluxuser test-infra/fixtures/userconfig-template.js /flux/ # may carry another (development does not track one), and a node started by pm2 # runs npm install at every start with no registry to reach: it succeeds only # when the lockfile and node_modules agree. -RUN cp /tmp/image-package-lock.json /flux/package-lock.json +RUN cp /tmp/image-package-lock.json /flux/package-lock.json && chown fluxuser:fluxuser /flux/package-lock.json RUN touch /flux/debug.log /flux/error.log /flux/info.log /flux/warn.log \ && chown fluxuser:fluxuser /flux /flux/debug.log /flux/error.log /flux/info.log /flux/warn.log \ From f220bc9a3b575a1817ba988675f65962ae227f92 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:23:08 +0100 Subject: [PATCH 237/369] fix(apps): the node status monitor starts once boot has settled, and boot's removals share its record of removals still running Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 6 +++-- .../appMonitoring/nodeStatusMonitor.js | 18 ++++++++++++- ZelBack/src/services/serviceManager.js | 4 +-- tests/unit/appStartupManager.test.js | 25 +++++++++++++++++++ tests/unit/nodeStatusMonitor.test.js | 24 ++++++++++++++++++ 5 files changed, 71 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 8a877ba949..6eaa15fb06 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -17,7 +17,8 @@ const appReconciler = require('../appMonitoring/appReconciler'); const appUninstaller = require('./appUninstaller'); const appNetworkLinker = require('./appNetworkLinker'); const appSlice = require('./appSlice'); -const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); +const nodeStatusMonitor = require('../appMonitoring/nodeStatusMonitor'); +const { TIMED_OUT } = require('../utils/withinDeadline'); const portManager = require('../appNetwork/portManager'); const globalState = require('../utils/globalState'); const fluxEventBus = require('../utils/fluxEventBus'); @@ -387,8 +388,9 @@ async function removeAllApps(reason) { log.warn(`REMOVAL REASON: ${reason} - removing ${app.name}`); // One removal that never returns must not hold boot, or every app after it. // eslint-disable-next-line no-await-in-loop - const outcome = await withinDeadline(appUninstaller.removeAppLocally(app.name, null, true, false, true), config.fluxapps.appRemovalDeadlineMs); + const outcome = await nodeStatusMonitor.removeWithinDeadline(appUninstaller.removeAppLocally, app.name, true); if (outcome === TIMED_OUT) log.error(`appStartupManager - ${app.name} was not removed within ${config.fluxapps.appRemovalDeadlineMs / 1000}s, moving on`); + else if (outcome === nodeStatusMonitor.STILL_REMOVING) log.warn(`appStartupManager - ${app.name} is still being removed, moving on`); } } } diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 09e2fbdff8..12b78780f6 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -14,6 +14,7 @@ const dockerService = require('../dockerService'); const { RemovalOutcome } = require('../utils/removalOutcome'); const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); const fluxEventBus = require('../utils/fluxEventBus'); +const globalState = require('../utils/globalState'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -32,7 +33,8 @@ const removalsRunning = new Set(); * Remove one app, waiting for it at most REMOVAL_DEADLINE_MS. A removal past * its deadline keeps running, so an app whose removal is still running is not * removed again: a second removal beside it would only wait behind the same - * stall, and the next pass would add a third. + * stall, and the next pass would add a third. Boot removes through this too, so a + * removal boot stopped waiting for is one the monitor does not start again. * @param {Function} removeAppLocallyFn * @param {string} name * @param {boolean} canBroadcast @@ -309,8 +311,22 @@ async function monitorNodeStatus(installedAppsFn, removeAppLocallyFn) { } } +/** + * Start the monitor once boot has settled, whatever boot decided: boot removes + * every app itself when the node may run none, and the two never remove at once. + * @param {Function} installedAppsFn + * @param {Function} removeAppLocallyFn + */ +async function start(installedAppsFn, removeAppLocallyFn) { + await globalState.waitForBootContainerStateSettled(); + monitorNodeStatus(installedAppsFn, removeAppLocallyFn); +} + module.exports = { initialize, + start, monitorNodeStatus, + removeWithinDeadline, unloadForHostStall, + STILL_REMOVING, }; diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index cce78a1508..55edc4d979 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -748,9 +748,7 @@ async function startFluxFunctions() { ); }, bootDelay(3 * 60 * 1000)); nodeStatusMonitor.initialize(appQueryService.installedApps, appUninstaller.removeAppLocally); - setTimeout(() => { - nodeStatusMonitor.monitorNodeStatus(appQueryService.installedApps, appUninstaller.removeAppLocally); - }, bootDelay(1.5 * 60 * 1000)); + nodeStatusMonitor.start(appQueryService.installedApps, appUninstaller.removeAppLocally); // Start the syncthing/masterSlave deciders once boot container state has settled // (the same AsyncGate the reconciler starts on), not after a fixed delay. Each // decider self-gates per cycle on its own prerequisites (mounts, syncthing health, diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 95f6141c28..2ed1db2b3f 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -19,6 +19,7 @@ describe('appStartupManager tests', () => { let appQueryServiceStub; let portManagerStub; let appSliceStub; + let nodeStatusMonitor; beforeEach(() => { appSliceStub = { stopEmptySlices: sinon.stub().resolves(0) }; @@ -91,6 +92,10 @@ describe('appStartupManager tests', () => { '../../lib/log': logStub, }); + // The real module, fresh per test: its record of removals still running is what + // boot shares with the monitor. + nodeStatusMonitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', {}); + appStartupManager = proxyquire('../../ZelBack/src/services/appLifecycle/appStartupManager', { '../../lib/log': logStub, '../dbHelper': dbHelperStub, @@ -103,6 +108,7 @@ describe('appStartupManager tests', () => { '../appMonitoring/appReconciler': appReconcilerStub, './appUninstaller': appUninstallerStub, './appSlice': appSliceStub, + '../appMonitoring/nodeStatusMonitor': nodeStatusMonitor, '../utils/globalState': globalStateStub, // runs the step again at once until it succeeds, as the real one does after a wait '../utils/untilSucceeds': { @@ -707,6 +713,25 @@ describe('appStartupManager tests', () => { } }); + it('leaves a removal it stopped waiting for to run, and the monitor does not start it again', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'stuck' }] }); + appUninstallerStub.removeAppLocally.withArgs('stuck').returns(new Promise(() => {})); + + const boot = appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); + await clock.tickAsync(config.fluxapps.appRemovalDeadlineMs); + await boot; + expect(globalStateStub.bootContainerStateSettled, 'fixture: boot moved past the removal').to.equal(true); + + const outcome = await nodeStatusMonitor.removeWithinDeadline(appUninstallerStub.removeAppLocally, 'stuck', true); + expect(outcome).to.equal(nodeStatusMonitor.STILL_REMOVING); + sinon.assert.calledOnce(appUninstallerStub.removeAppLocally); + } finally { + clock.restore(); + } + }); + it('stops every empty app slice before it removes any app', async () => { appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 5849f79338..9f03368c15 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -69,6 +69,30 @@ describe('nodeStatusMonitor tests', () => { // Full testing would require integration tests or extensive mocking }); + describe('start', () => { + it('runs no pass until boot has settled, and runs one once it has', async () => { + let settle; + const settled = new Promise((resolve) => { settle = resolve; }); + const isNodeDos = sinon.stub().returns(false); + const monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { + '../serviceHelper': { delay: () => new Promise(() => {}) }, + '../fluxNetworkHelper': { isNodeDos }, + '../nodeConfirmationService': { isDaemonStale: () => false, isConfirmed: () => false }, + '../utils/globalState': { waitForBootContainerStateSettled: () => settled }, + '../../lib/log': logStub, + }); + const installedAppsFn = sinon.stub().resolves({ status: 'success', data: [] }); + + const started = monitor.start(installedAppsFn, sinon.stub()); + await new Promise((resolve) => { setImmediate(resolve); }); + sinon.assert.notCalled(isNodeDos); + + settle(); + await started; + sinon.assert.calledOnce(isNodeDos); + }); + }); + describe('initialize', () => { let listeners; let monitor; From cfe128680194ae94152d62a0f48cf457fb7e3e2c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:30:27 +0100 Subject: [PATCH 238/369] test(harness): 2208 reads and drives pm2 as the operator's account FluxOS runs under Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/container.js | 7 ++++--- ...gacy-pm2-kill-timeout-request-never-strands-the-boot.js | 6 +++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/test-infra/runner/framework/container.js b/test-infra/runner/framework/container.js index b5e7e7bae1..5e00d6d335 100644 --- a/test-infra/runner/framework/container.js +++ b/test-infra/runner/framework/container.js @@ -455,9 +455,10 @@ export async function crashFluxos(container, { hold = false, exitTimeoutMs = 300 throw new Error(`crashFluxos: FluxOS ${pid} still running ${exitTimeoutMs}ms after SIGKILL`); } -// pm2 as a legacy node's operator runs it: as their account, against their pm2 home -// (test-infra/systemd/pm2-fluxuser.service). -const PM2 = 'runuser -u fluxuser -- env PM2_HOME=/home/fluxuser/.pm2 pm2'; +// A command as a legacy node's operator runs pm2: as their account, against their pm2 +// home (test-infra/systemd/pm2-fluxuser.service). +export const AS_PM2_OPERATOR = 'runuser -u fluxuser -- env PM2_HOME=/home/fluxuser/.pm2'; +const PM2 = `${AS_PM2_OPERATOR} pm2`; // The kill timeout of FluxOS's pm2 registration on a legacy node whose FluxOS // pm2 runs (createTestEnv pm2Nodes), as the daemon holds it and as `pm2 save` diff --git a/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js b/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js index 0cc93653a6..b3f5680fbb 100644 --- a/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js +++ b/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js @@ -1,7 +1,7 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { execInContainer, pm2Registration } from '../framework/container.js'; +import { execInContainer, pm2Registration, AS_PM2_OPERATOR } from '../framework/container.js'; import { bootAndPeer } from '../framework/reconciler-suite.js'; import { waitFor } from '../framework/wait.js'; import { isDaemonUp } from '../framework/syncthing-real.js'; @@ -32,7 +32,7 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // request in the boot. const SHORT_MS = 1600; -const REQUEST_MARKER = '/root/.flux-pm2-kill-timeout-request'; +const REQUEST_MARKER = '/home/fluxuser/.flux-pm2-kill-timeout-request'; // How long a restart loop would take to show itself: pm2 restarts FluxOS // within seconds of being asked. const QUIET_MS = 45000; @@ -57,7 +57,7 @@ describe('a legacy node\'s request for a longer pm2 kill timeout never strands i // The registration as the multitool made it before it set a kill timeout, // with no request recorded for this boot: FluxOS asks as it restarts. const registerShort = async () => { - const r = await sh(`rm -f ${REQUEST_MARKER} && ${realPm2} restart flux --kill-timeout ${SHORT_MS} >/dev/null && ${realPm2} save >/dev/null`); + const r = await sh(`rm -f ${REQUEST_MARKER} && ${AS_PM2_OPERATOR} ${realPm2} restart flux --kill-timeout ${SHORT_MS} >/dev/null && ${AS_PM2_OPERATOR} ${realPm2} save >/dev/null`); expect(r.exitCode, `fixture: ${r.output}`).to.equal(0); }; From b445e5c51a180e309c9dfc24fe30033297959f6a Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:37:39 +0100 Subject: [PATCH 239/369] fix(apps): a host stall takes the node out of service whether or not its trip can be recorded, and a start does not wait on a record it cannot read Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 3 ++ ZelBack/src/services/hostStallService.js | 17 +++++-- tests/unit/globalconfig/default.js | 3 ++ tests/unit/hostStallService.test.js | 64 ++++++++++++++++++++++++ 4 files changed, 84 insertions(+), 3 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index df3e05bc7e..dd21e70900 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -50,6 +50,9 @@ module.exports = { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, stopWindowMs: 60 * 1000, + // How long a start waits to read the trip record. A healthy disk reads it in + // milliseconds. + recordReadMs: 10 * 1000, }, database: { url: '127.0.0.1', diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 58fd4dfb9e..404ec9586a 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -28,10 +28,12 @@ const config = require('config'); const log = require('../lib/log'); const fluxNetworkHelper = require('./fluxNetworkHelper'); const { fluxDirPath } = require('./utils/appConstants'); +const { withinDeadline, TIMED_OUT } = require('./utils/withinDeadline'); const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); const READ_INTERVAL_MS = config.hostStall.readIntervalMs; const STUCK_INTERVALS = config.hostStall.stuckIntervals; +const RECORD_READ_MS = config.hostStall.recordReadMs; /** * Every dependency thread in D at the last reading, keyed by process, thread @@ -237,8 +239,10 @@ async function act() { if (longest.intervals >= STUCK_INTERVALS && !fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { const reason = `Host stall: ${describe(longest)}`; fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); - await recordTrip(reason); + // Announced before it is recorded, and the reading does not wait for the record: + // the disk the record goes to may be the one that stalled. announceTrip(); + recordTrip(reason); } } @@ -267,14 +271,21 @@ async function readAndReschedule() { /** * Restore a trip recorded in this boot, then read now, and every * READ_INTERVAL_MS after each reading ends. + * + * A record that cannot be read within RECORD_READ_MS is not waited for further: + * its disk is stalled, and a stall still under way takes the node out of service + * again at the readings. A restore that ends later still restores. * @param {object} [options] Paths, as usePaths takes them. - * @returns {Promise} Resolves once a recorded trip is restored. + * @returns {Promise} Resolves once a recorded trip is restored, or the + * record could not be read within RECORD_READ_MS. */ async function start(options = {}) { if (readTimer !== null) return; usePaths(options); readTimer = setTimeout(readAndReschedule, 0); - await restoreTrip(); + if (await withinDeadline(restoreTrip(), RECORD_READ_MS) === TIMED_OUT) { + log.error(`hostStall - the trip record could not be read within ${RECORD_READ_MS / 1000}s; going on without it`); + } } /** Take no more readings and forget every thread. */ diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index bc7a836e61..7482ab75bb 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -33,6 +33,9 @@ module.exports = { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, stopWindowMs: 60 * 1000, + // How long a start waits to read the trip record. A healthy disk reads it in + // milliseconds. + recordReadMs: 10 * 1000, }, database: { url: database, diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 20afae90b5..3add177cb9 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -227,6 +227,7 @@ describe('hostStallService tests', () => { it('tells a listener when a trip recorded in this boot is restored at start', async () => { stuckMongod(); await readTimes(stuckIntervals + 1); + while (!fs.existsSync(paths.recordFile)) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop hostStallService.stop(); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); const listener = sinon.stub(); @@ -239,9 +240,21 @@ describe('hostStallService tests', () => { }); describe('the trip record', () => { + const settled = () => new Promise((resolve) => { setImmediate(resolve); }); + // The trip, and its record on disk where the boot id says one is made. async function trip() { stuckMongod(); await readTimes(stuckIntervals + 1); + while (fs.existsSync(paths.bootIdPath) && !fs.existsSync(paths.recordFile)) await settled(); // eslint-disable-line no-await-in-loop + } + // Every write of the record to disk, held until released. + function holdRecordWrites() { + const writeFile = fs.promises.writeFile.bind(fs.promises); + const held = []; + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.recordFile) + ? new Promise((resolve) => { held.push(() => resolve(writeFile(file, ...rest))); }) + : writeFile(file, ...rest))); + return held; } async function restart() { hostStallService.stop(); @@ -294,6 +307,57 @@ describe('hostStallService tests', () => { expect(fs.existsSync(paths.recordFile)).to.equal(true); }); + it('takes the node out of service and tells its listeners while the record cannot be written', async () => { + holdRecordWrites(); + const listener = sinon.stub(); + hostStallService.onTrip(listener); + stuckMongod(); + + await readTimes(stuckIntervals + 1); + + expect(outOfService()).to.equal(true); + sinon.assert.calledOnce(listener); + }); + + it('goes on reading while the record cannot be written', async () => { + holdRecordWrites(); + stuckMongod(); + await readTimes(stuckIntervals + 1); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 41 }]); + + const outcome = await Promise.race([read().then(() => 'read'), new Promise((resolve) => { setTimeout(() => resolve('waiting'), 1000); })]); + + expect(outcome, 'a reading waited on the record').to.equal('read'); + expect(placementHeld(), 'the reading after the trip released the hold').to.equal(false); + }); + + it('starts without the record when it cannot be read in time, and restores it when the read ends', async () => { + await trip(); + hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + const readFile = fs.promises.readFile.bind(fs.promises); + let finishRead = null; + sinon.stub(fs.promises, 'readFile').callsFake((file, ...rest) => (file === paths.recordFile + ? new Promise((resolve) => { finishRead = () => resolve(readFile(file, ...rest)); }) + : readFile(file, ...rest))); + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + const started = hostStallService.start(paths); + let done = false; + started.then(() => { done = true; }); + await clock.tickAsync(config.hostStall.recordReadMs - 1); + expect(done, 'the start returned before its wait for the record ran out').to.equal(false); + await clock.tickAsync(1); + await started; + expect(outOfService(), 'restored without a record').to.equal(false); + + finishRead(); + while (!outOfService()) await clock.tickAsync(0); // eslint-disable-line no-await-in-loop + } finally { + clock.restore(); + } + }); + it('starts in service when the record cannot be read', async () => { fs.writeFileSync(paths.recordFile, '{ not json'); From 6ae5b427487e824d463ad8323da1c1ab2ee5df1e Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:39:22 +0100 Subject: [PATCH 240/369] test(harness): 3303, a node whose config directory stalls with docker stops its apps, and records the trip once the directory takes writes Co-Authored-By: Claude Opus 5.5 --- test-infra/entrypoint.sh | 16 +++++ test-infra/runner/framework/host-stall.js | 20 +++++- test-infra/runner/framework/test-env.js | 12 +++- ...lled-host-takes-the-node-out-of-service.js | 71 ++++++++++++++++++- 4 files changed, 111 insertions(+), 8 deletions(-) diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 16e5568fc4..3b4c067457 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -63,6 +63,22 @@ if [ "$FLUX_FREEZABLE_DOCKER_ROOT" = "true" ]; then mountpoint -q /mnt/appdata/docker || mount -o loop "$img" /mnt/appdata/docker fi +# The same for the node's FluxOS config directory, where FluxOS keeps the record of +# a host stall: frozen, every write FluxOS makes there is held in the kernel. Seeded +# with the directory's own files and owner on first boot. +if [ "$FLUX_FREEZABLE_CONFIG" = "true" ]; then + cfg="${FLUXOS_PATH:-/flux}/config" + img=/mnt/appdata/flux-config.img + owner="$(stat -c %u:%g "$cfg")" + if [ ! -f "$img" ]; then + truncate -s 64M "$img" && mkfs.ext4 -q -F "$img" + seed="$(mktemp -d)" + mount -o loop "$img" "$seed" && cp -a "$cfg/." "$seed/" && chown "$owner" "$seed" && umount "$seed" + rmdir "$seed" + fi + mountpoint -q "$cfg" || mount -o loop "$img" "$cfg" +fi + # In stub mode FluxOS only needs somewhere to read an API key from; the calls # themselves go to the shared stub. In binary mode the config is syncthing's own # and this fixture must not be in the way of it. diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index d1caedf8a1..880dccefbd 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -9,6 +9,8 @@ import { execInContainer } from './container.js'; export const DOCKER_ROOT = '/mnt/appdata/docker'; +// A harness node's FluxOS directory is /flux, legacy and Arcane alike. +export const FLUXOS_CONFIG = '/flux/config'; async function run(container, command, what) { const result = await execInContainer(container, command); @@ -29,14 +31,28 @@ export async function thawDocker(container) { } /** - * Thaw every node's docker data root that is frozen, and every cgroup frozen in it + * Hold every write the node's FluxOS makes to its config directory, until + * thawConfig. The node is booted with createTestEnv({ freezableConfig: [index] }). + */ +export async function freezeConfig(container) { + await run(container, `mountpoint -q ${FLUXOS_CONFIG}`, `${FLUXOS_CONFIG} is not a mount of its own: boot the node with freezableConfig`); + await run(container, `fsfreeze -f ${FLUXOS_CONFIG}`, 'freezeConfig'); +} + +/** Let the node's FluxOS write its config directory again. One not frozen is left as it is. */ +export async function thawConfig(container) { + await run(container, `out=$(fsfreeze -u ${FLUXOS_CONFIG} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }`, 'thawConfig'); +} + +/** + * Thaw every node's docker data root and config directory that is frozen, and every cgroup frozen in it * (systemd-control.freezeUnit), and pass over the rest: the last step before * anything reads a node through its docker or stops it, since a node whose docker * is held can be neither read nor stopped. Never rejects. * @param {Array<{container?: object}>} clients */ export async function thawEveryNode(clients) { - const thaw = `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; ` + const thaw = `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; fsfreeze -u ${FLUXOS_CONFIG} 2>/dev/null; ` + 'for f in $(find /sys/fs/cgroup -name cgroup.freeze 2>/dev/null); do [ "$(cat $f)" = 1 ] && echo 0 > $f; done; true'; await Promise.all(clients.map(async (client) => { if (!client?.container) return; diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 5bcfbb20ef..f1f4f5727c 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -971,7 +971,7 @@ export async function createTestEnv({ tickerAutostart = false, discoveryAutostart = false, nodeStatusOverrides = {}, rpcFailures = [], bootContext = 'running', initialHeight = DEFAULT_INITIAL_HEIGHT, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation = {}, locationTable = null, staticIp = true, policy = null, policySeeds = null, - awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], + awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], freezableConfig = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], // Firewalled nodes boot with ufw active as their install leaves it - the legacy // installer's rules on a legacy node, the ISO's on an Arcane one - and FluxOS // adds its own on top. true for every node, or a list of node indices. Off @@ -988,6 +988,11 @@ export async function createTestEnv({ for (const index of freezableDockerRoot) { if (!Number.isInteger(index) || index < 0 || index >= nodes) throw new Error(`createTestEnv: freezableDockerRoot names node ${index}, outside a fleet of ${nodes}`); } + // Nodes whose FluxOS config directory is a filesystem a suite can freeze + // (framework/host-stall.js), by node index. + for (const index of freezableConfig) { + if (!Number.isInteger(index) || index < 0 || index >= nodes) throw new Error(`createTestEnv: freezableConfig names node ${index}, outside a fleet of ${nodes}`); + } for (const index of pausedSyncthingNodes) { if (syncthing !== 'binary') throw new Error('createTestEnv: pausedSyncthingNodes needs syncthing: \'binary\''); if (legacyNodes.includes(index)) throw new Error(`createTestEnv: paused syncthing node ${index} is legacy, whose syncthing FluxOS starts`); @@ -1278,7 +1283,7 @@ export async function createTestEnv({ // mongo starts, i.e. inside the fleet boot, where the waits at risk are the // boot's own. await startInfraDeathWatch(env); - await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot); + await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot, freezableConfig); return env; } catch (err) { // Boot failed: the env owns everything started so far. The shared teardown @@ -1307,7 +1312,7 @@ function mergeConfigs(base, override) { return result; } -async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = []) { +async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], freezableConfig = []) { // Everything built here registers onto the env shell as it comes up, so a // boot-phase throw leaves the partial state reachable (see makeEnvShell). const { @@ -1674,6 +1679,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod if (!isLegacy) nodeEnv.FLUXOS_PATH = '/flux'; if (pausedSyncthingNodes.includes(i)) nodeEnv.FLUX_SYNCTHING_PAUSED = '1'; if (freezableDockerRoot.includes(i)) nodeEnv.FLUX_FREEZABLE_DOCKER_ROOT = 'true'; + if (freezableConfig.includes(i)) nodeEnv.FLUX_FREEZABLE_CONFIG = 'true'; if (syncthingFaults[i]) nodeEnv.FLUX_SYNCTHING_FAULT = syncthingFaults[i]; // WHETHER THIS NODE'S FLUXOS IS ROOT. Declared per node rather than derived from // legacy, because the field carries both: Arcane runs FluxOS as root, and an diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 06b8120216..3df40a35e1 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -2,12 +2,15 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { execInContainer, getAppContainerStatus } from '../framework/container.js'; -import { waitForUp, waitForAppInstalled, assertNoEvent } from '../framework/wait.js'; +import { + waitFor, waitForUp, waitForAppInstalled, assertNoEvent, +} from '../framework/wait.js'; import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { buildSeedableTestApp } from '../framework/seed-helper.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { - freezeDocker, thawDocker, thawEveryNode, stuckThreads, processesThatCanRun, containerCgroup, + freezeDocker, thawDocker, freezeConfig, thawConfig, FLUXOS_CONFIG, + thawEveryNode, stuckThreads, processesThatCanRun, containerCgroup, } from '../framework/host-stall.js'; import { DOCKERD_UNIT, freezeUnit, restartFluxos, unitCgroup, @@ -28,7 +31,9 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // runs out. With docker frozen as well (systemd-control.freezeUnit), docker // cannot act at all, and the app is killed without docker. A node holding several // apps has every one of them stopped within that one window, long before even one -// removal's deadline. +// removal's deadline. A node whose FluxOS config directory, where the trip is +// recorded, is frozen as well stops its apps all the same, and records the trip +// once the directory takes writes again. // // The nodes run systemd, so docker uses its systemd cgroup driver and every app's // containers live in the app's own slice, as on a production node. The node the @@ -90,6 +95,7 @@ describe('a stalled host takes the node out of service', function () { nodes: 4, tickerAutostart: false, freezableDockerRoot: [0, 1, 2, 3], + freezableConfig: [0, 1, 2, 3], legacyNodes: [LEGACY_UNPRIVILEGED], unprivilegedNodes: [LEGACY_UNPRIVILEGED], configOverrides: { @@ -332,4 +338,63 @@ describe('a stalled host takes the node out of service', function () { .forEach((a) => expect(a.runnable, `the node reports a process of ${a.app} that can still run`).to.deep.equal([])); }); }); + + describe('a stall past the threshold, on a node whose config directory stalls too', function () { + const recordName = `e2ehoststallrec${Date.now()}`; + const record = `${FLUXOS_CONFIG}/hostStall.json`; + let recClient; + let recContainer; + let anchor; + let trippedAt; + let stopped; + let recordedWhileFrozen; + + before(async function () { + this.timeout(480000); + // the Arcane node no other case here uses + const index = ARCANE.filter((i) => i !== node).at(-1); + recClient = env.clients[index]; + ({ container: recContainer } = recClient); + await installOn(env, recordName, [index]); + await waitForUp(recClient, recordName, `${recordName} is up`); + anchor = recClient.getLastEventId(); + step(`freezing node ${index}'s docker and its config directory`); + await freezeConfig(recContainer); + await freezeDocker(recContainer); + const trip = await recClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + stopped = (await recClient.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + recordedWhileFrozen = (await execInContainer(recContainer, `test -e ${record}`)).exitCode === 0; + }); + + after(async function () { + if (recClient) await thawEveryNode([recClient]); + }); + + it('held the config directory: a write to it does not return', async function () { + const write = await execInContainer(recContainer, `timeout 3 touch ${FLUXOS_CONFIG}/.probe`); + expect(write.exitCode, 'a write to the config directory returned, so the record was never held').to.equal(124); + }); + + it('takes the node out of service', function () { + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('stops its apps though the trip cannot be recorded', function () { + expect(recordedWhileFrozen, 'fixture: the trip was recorded on a frozen directory').to.equal(false); + expect(stopped, 'the node never stopped its apps while its trip could not be recorded').to.not.equal(null); + expect(stopped.apps.map((a) => a.app), 'the apps the node stopped').to.include(recordName); + }); + + it('records the trip once the config directory takes writes again', async function () { + this.timeout(60000); + await thawConfig(recContainer); + await thawDocker(recContainer); + await waitFor(async () => (await execInContainer(recContainer, `grep -q 'Host stall' ${record}`)).exitCode === 0, { + timeout: 30000, interval: 1000, label: 'the trip recorded once the config directory came back', + }); + }); + }); }); From ce7b578d8f3829ae8d3fcce626ab7dc286f0da9a Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:55:01 +0100 Subject: [PATCH 241/369] test(harness): 3303 reads a held config write from the background, since a write held on a frozen filesystem cannot be ended Co-Authored-By: Claude Opus 5.5 --- .../3303-a-stalled-host-takes-the-node-out-of-service.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 3df40a35e1..683ca91795 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -374,8 +374,11 @@ describe('a stalled host takes the node out of service', function () { }); it('held the config directory: a write to it does not return', async function () { - const write = await execInContainer(recContainer, `timeout 3 touch ${FLUXOS_CONFIG}/.probe`); - expect(write.exitCode, 'a write to the config directory returned, so the record was never held').to.equal(124); + // A write held on a frozen filesystem cannot be killed, so it is left to finish + // in the background, after the thaw. + const write = await execInContainer(recContainer, + `rm -f /tmp/probe-written; setsid sh -c 'touch ${FLUXOS_CONFIG}/.probe && touch /tmp/probe-written' /dev/null 2>&1 & sleep 3; test ! -e /tmp/probe-written`); + expect(write.exitCode, 'a write to the config directory returned, so the record was never held').to.equal(0); }); it('takes the node out of service', function () { From 47f230df6f331ebb4e0411ffa11d4562aa24aee2 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:08:37 +0100 Subject: [PATCH 242/369] fix(apps): a stall's kill finds an app's containers by their app label, and by FluxOS's own naming where a container has none Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 47 +++++++++++++------- tests/unit/appKill.test.js | 38 +++++++++++++++- 2 files changed, 66 insertions(+), 19 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index 3c022623d1..b4aeddbf50 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -20,6 +20,7 @@ const path = require('path'); const log = require('../../lib/log'); const serviceHelper = require('../serviceHelper'); const appSlice = require('./appSlice'); +const { getAppIdentifier } = require('../dockerService'); const { isArcane } = require('../utils/appConstants'); const SIGKILL_MASK = 1n << 8n; @@ -50,28 +51,38 @@ async function readAsRoot(file) { return error ? null : stdout; } +// The name FluxOS gives an app's containers carries the flux or zel prefix +// (dockerService.getAppIdentifier). +const APP_PREFIX = /^(flux|zel)/; + /** - * The app a container belongs to: `flux` for a single container, - * `flux_` for a component. - * @param {string} name The container's name. + * The app a container belongs to: its runonflux.app label, which FluxOS sets on + * every container it creates; for a container made before the label, read from + * its name, `_` or `flux`. A label-less container + * of an app whose own name starts with flux reads as the name without it. + * @param {{name: string, app: string|null}} container * @returns {string|null} Null for a container that is not an app's. */ -function appOf(name) { - if (!name.startsWith('flux')) return null; - return name.includes('_') ? name.slice(name.indexOf('_') + 1) : name.slice('flux'.length); +function appOf({ name, app }) { + if (app) return app; + if (!APP_PREFIX.test(name)) return null; + if (name.includes('_')) return name.slice(name.indexOf('_') + 1); + return name.startsWith('flux') ? name.slice('flux'.length) : name; } /** - * @param {string} name The container's name. + * @param {{name: string, app: string|null}} container * @param {string} appName */ -function belongsTo(name, appName) { - return appOf(name) === appName; +function belongsTo({ name, app }, appName) { + if (app) return app === appName; + return name === getAppIdentifier(appName) || (APP_PREFIX.test(name) && name.endsWith(`_${appName}`)); } /** * Every container docker has a record of, read from its records on disk. - * @returns {Promise>} Names without the leading '/'. + * @returns {Promise>} Names + * without the leading '/', and the app its runonflux.app label names. */ async function containersOnDisk() { const listing = await serviceHelper.runCommand('ls', { runAsRoot: true, params: ['-1', path.join(paths.dockerRoot, 'containers')] }); @@ -82,13 +93,15 @@ async function containersOnDisk() { for (const id of ids) { // eslint-disable-next-line no-await-in-loop const text = await readAsRoot(path.join(paths.dockerRoot, 'containers', id, 'config.v2.json')); - let name = null; + let parsed = null; try { - ({ Name: name } = JSON.parse(text ?? 'null') ?? {}); + parsed = JSON.parse(text ?? 'null'); } catch { - name = null; + parsed = null; } - if (typeof name === 'string') containers.push({ id, name: name.replace(/^\//, '') }); + const name = parsed?.Name; + const app = parsed?.Config?.Labels?.['runonflux.app']; + if (typeof name === 'string') containers.push({ id, name: name.replace(/^\//, ''), app: typeof app === 'string' && app ? app : null }); } return containers; } @@ -99,7 +112,7 @@ async function containersOnDisk() { * @returns {Promise} */ async function containersOf(appName) { - return (await containersOnDisk()).filter(({ name }) => belongsTo(name, appName)).map(({ name }) => name); + return (await containersOnDisk()).filter((container) => belongsTo(container, appName)).map(({ name }) => name); } /** @@ -109,7 +122,7 @@ async function containersOf(appName) { */ async function scopesOf(appName) { return (await containersOnDisk()) - .filter(({ name }) => belongsTo(name, appName)) + .filter((container) => belongsTo(container, appName)) .map(({ id }) => `docker-${id}.scope`); } @@ -195,7 +208,7 @@ async function appsOnDisk() { const slices = parent ? await fs.readdir(parent).catch(() => []) : []; slices.filter((entry) => /^flux-.+\.slice$/.test(entry)) .forEach((entry) => names.add(entry.slice('flux-'.length, -'.slice'.length).replace(/\\x2d/g, '-'))); - (await containersOnDisk()).map(({ name }) => appOf(name)).filter(Boolean).forEach((name) => names.add(name)); + (await containersOnDisk()).map(appOf).filter(Boolean).forEach((name) => names.add(name)); return [...names]; } diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index b5036170b8..94197ace58 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -28,9 +28,11 @@ describe('appKill tests', () => { if (rel.startsWith('flux.slice/')) units.set('flux.slice', path.join(SYSTEMD_ROOT, 'flux.slice')); return dir; } - function record(n, name, pid = 0) { + // `app`: the runonflux.app label FluxOS sets; left out, a container made before it + function record(n, name, pid = 0, { app } = {}) { fs.mkdirSync(path.join(dockerRoot, 'containers', id(n)), { recursive: true }); - fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pid) } })); + const labels = app ? { 'runonflux.app': app } : {}; + fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pid) }, Config: { Labels: labels } })); } function container(n, name, pids) { record(n, name, pids[0]); @@ -158,6 +160,13 @@ describe('appKill tests', () => { expect(await appKill.appsOnDisk()).to.have.members(['sliced-app', 'composed', 'single']); }); + + it('names a labelled container\'s app by its label, and a zel component\'s by its name', async () => { + record(1, 'fluxgame', 0, { app: 'fluxgame' }); + record(2, 'zelcash_legacy'); + + expect(await appKill.appsOnDisk()).to.have.members(['fluxgame', 'legacy']); + }); }); describe('containersOf', () => { @@ -169,5 +178,30 @@ describe('appKill tests', () => { expect(await appKill.containersOf('composed')).to.have.members(['fluxweb_composed', 'fluxdb_composed', 'fluxcomposed']); }); + + it('takes a container\'s label over its name', async () => { + record(1, 'fluxweb_composed', 0, { app: 'other' }); + record(2, 'renamed', 0, { app: 'composed' }); + + expect(await appKill.containersOf('composed')).to.have.members(['renamed']); + }); + + it('finds a label-less component whose name starts with zel', async () => { + record(1, 'zelcash_legacy'); + + expect(await appKill.containersOf('legacy')).to.have.members(['zelcash_legacy']); + }); + + it('finds the label-less container of an app whose own name starts with flux', async () => { + record(1, 'fluxgame'); + + expect(await appKill.containersOf('fluxgame')).to.have.members(['fluxgame']); + }); + + it('does not take a container FluxOS did not name for an app whose name it ends with', async () => { + record(1, 'my_composed'); + + expect(await appKill.containersOf('composed')).to.deep.equal([]); + }); }); }); From ad0359e732a5fd6bd2f9c55f651cba22c4eed58d Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:13:20 +0100 Subject: [PATCH 243/369] fix(apps): an empty app slice's stop is queued with systemd, not waited for Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appSlice.js | 7 +++++-- tests/unit/appSlice.test.js | 4 ++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appSlice.js b/ZelBack/src/services/appLifecycle/appSlice.js index d2a5152a53..9740a91eeb 100644 --- a/ZelBack/src/services/appLifecycle/appSlice.js +++ b/ZelBack/src/services/appLifecycle/appSlice.js @@ -71,12 +71,15 @@ async function isPopulated(unit) { /** * Stop a slice that holds no process. A slice in use is left alone. + * + * The stop is queued and not waited for: an empty slice has nothing to stop, and + * systemd's own bus timeout bounds the request. * @param {string} unit - * @returns {Promise} True when the slice was stopped. + * @returns {Promise} True when systemd took the stop. */ async function stopIfEmpty(unit) { if (await isPopulated(unit) !== false) return false; - const { error } = await serviceHelper.runCommand('systemctl', { runAsRoot: true, params: ['stop', unit] }); + const { error } = await serviceHelper.runCommand('systemctl', { runAsRoot: true, params: ['stop', '--no-block', unit] }); if (error) { log.error(`appSlice - ${unit} could not be stopped: ${error.message}`); return false; diff --git a/tests/unit/appSlice.test.js b/tests/unit/appSlice.test.js index 5dba6961ed..e16338f59e 100644 --- a/tests/unit/appSlice.test.js +++ b/tests/unit/appSlice.test.js @@ -22,7 +22,7 @@ describe('appSlice tests', () => { units.set(unit, path.join(SYSTEMD_ROOT, 'flux.slice', unit)); } const stops = () => runCommand.getCalls().filter((call) => call.args[1].params[0] === 'stop'); - const stopped = () => stops().map((call) => call.args[1].params[1]); + const stopped = () => stops().map((call) => call.args[1].params.at(-1)); let stopResult; beforeEach(() => { @@ -64,7 +64,7 @@ describe('appSlice tests', () => { await appSlice.release('my-app'); - expect(stops().map((call) => call.args)).to.deep.equal([['systemctl', { runAsRoot: true, params: ['stop', 'flux-my\\x2dapp.slice'] }]]); + expect(stops().map((call) => call.args)).to.deep.equal([['systemctl', { runAsRoot: true, params: ['stop', '--no-block', 'flux-my\\x2dapp.slice'] }]]); }); it('leaves a slice a process still lives in', async () => { From b466143706a267a147498d7177688094b6d0f2df Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 06:40:30 +0100 Subject: [PATCH 244/369] fix(apps): a host stall's unload reads docker's records once, within the stop window The unload reads docker's container records once and passes them to every step: finding the apps, asking docker to stop them, checking and killing them. A disk that does not answer reads holds that read in the kernel, so it is waited for at most hostStall.stopWindowMs; every app is then still stopped and killed through its slice, which the kernel reports without the disk. Without the records a container outside the app slices cannot be named. An app with nothing left in its slice is then reported with runnable null, never as unable to run, unless no container lives outside the slices at all. A listing docker's records refuses reads the same way. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 99 ++++++++++++------- .../appMonitoring/nodeStatusMonitor.js | 37 ++++--- tests/unit/appKill.test.js | 83 +++++++++++++--- tests/unit/nodeStatusMonitor.test.js | 57 ++++++++++- 4 files changed, 210 insertions(+), 66 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index b4aeddbf50..15a45733af 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -2,12 +2,16 @@ * Kill every process of an app through its cgroups, without docker or the * database, and say whether any of them can still run. * - * An app's containers live in its slice (appSlice). A container created before - * the slice existed lives in its runtime's scope, `docker-.scope`, and the - * container's name is read from the runtime's own record on disk, which stays - * readable while docker does not answer. Where those records are is read from - * docker at boot; until then it is docker's default for the node's kind. Where - * each cgroup is, systemd says (appSlice.controlGroup). + * An app's containers live in its slice (appSlice), found from the kernel alone. + * A container created before the slice existed lives in its runtime's scope, + * `docker-.scope`, outside any app's slice, and which app it belongs to is + * read from the runtime's own records on disk (readContainers), which stay + * readable while docker does not answer but not while its disk does not. The + * records are read once and passed to every call here; null stands for records + * that could not be read, and then a container outside the slices cannot be + * named, killed or ruled out. Where the records are is read from docker at boot; + * until then it is docker's default for the node's kind. Where each cgroup is, + * systemd says (appSlice.controlGroup). * * A process held in the kernel is not removed by a kill: it keeps the kill * pending and dies the moment the kernel lets it go, without running the app's @@ -80,13 +84,18 @@ function belongsTo({ name, app }, appName) { } /** - * Every container docker has a record of, read from its records on disk. - * @returns {Promise>} Names - * without the leading '/', and the app its runonflux.app label names. + * @typedef {{id: string, name: string, app: string|null}} ContainerRecord + * The name without its leading '/', and the app its runonflux.app label names. */ -async function containersOnDisk() { + +/** + * Every container docker has a record of, read from its records on disk. Nothing + * bounds the read: on a disk that does not answer it never returns. + * @returns {Promise} Null when the records cannot be listed. + */ +async function readContainers() { const listing = await serviceHelper.runCommand('ls', { runAsRoot: true, params: ['-1', path.join(paths.dockerRoot, 'containers')] }); - if (listing.error) return []; + if (listing.error) return null; const ids = listing.stdout.split('\n').map((line) => line.trim()).filter((id) => /^[0-9a-f]{64}$/.test(id)); const containers = []; // eslint-disable-next-line no-restricted-syntax @@ -107,33 +116,35 @@ async function containersOnDisk() { } /** - * The names of the app's containers, read from docker's records on disk. * @param {string} appName - * @returns {Promise} + * @param {ContainerRecord[]|null} containers readContainers + * @returns {ContainerRecord[]} Empty when the records could not be read. */ -async function containersOf(appName) { - return (await containersOnDisk()).filter((container) => belongsTo(container, appName)).map(({ name }) => name); +function recordsOf(appName, containers) { + return (containers ?? []).filter((container) => belongsTo(container, appName)); } /** - * The scope units of the app's containers, whether in its slice or not. + * The names of the app's containers. * @param {string} appName - * @returns {Promise} + * @param {ContainerRecord[]|null} containers readContainers + * @returns {string[]} Empty when the records could not be read. */ -async function scopesOf(appName) { - return (await containersOnDisk()) - .filter((container) => belongsTo(container, appName)) - .map(({ id }) => `docker-${id}.scope`); +function containersOf(appName, containers) { + return recordsOf(appName, containers).map(({ name }) => name); } /** - * Every cgroup that holds a container of the app and exists now. + * Every cgroup that holds a container of the app and exists now: its slice, and + * the scope of each of its containers the records name. * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers * @returns {Promise} */ -async function cgroupsOf(appName) { +async function cgroupsOf(appName, containers) { const unit = appSlice.sliceName(appName); - const units = [...(unit ? [unit] : []), ...await scopesOf(appName)]; + const scopes = recordsOf(appName, containers).map(({ id }) => `docker-${id}.scope`); + const units = [...(unit ? [unit] : []), ...scopes]; const found = (await Promise.all(units.map(appSlice.controlGroup))).filter(Boolean); // a scope inside the app's slice is read and killed with the slice const outermost = [...new Set(found)].filter((dir, _, all) => !all.some((other) => dir.startsWith(`${other}/`))); @@ -144,10 +155,11 @@ async function cgroupsOf(appName) { /** * Send SIGKILL to every process of the app. * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers * @returns {Promise} The cgroups it was sent to. */ -async function kill(appName) { - const cgroups = await cgroupsOf(appName); +async function kill(appName, containers) { + const cgroups = await cgroupsOf(appName, containers); // eslint-disable-next-line no-restricted-syntax for (const dir of cgroups) { // eslint-disable-next-line no-await-in-loop @@ -184,31 +196,49 @@ async function processesUnder(dir) { return [...own, ...below.flat()]; } +/** + * Whether any container lives outside the app slices: in a scope directly under + * system.slice, where the runtime puts a container with no slice of its own. + * @returns {Promise} True as well when systemd does not answer. + */ +async function anyContainerOutsideSlices() { + const parent = await appSlice.controlGroup('system.slice'); + if (!parent) return true; + const entries = await fs.readdir(parent).catch(() => null); + return entries === null || entries.some((entry) => /^docker-[0-9a-f]{64}\.scope$/.test(entry)); +} + /** * Which of the app's processes could still run its code: every one that is * neither gone nor has SIGKILL pending. * @param {string} appName - * @returns {Promise} Their pids; empty when the app can never run again. + * @param {ContainerRecord[]|null} containers readContainers + * @returns {Promise} Their pids; empty when the app can never run + * again; null when none was found but the records could not be read and a + * container outside the slices, which may be the app's, exists. */ -async function stillRunnable(appName) { - const cgroups = await cgroupsOf(appName); +async function stillRunnable(appName, containers) { + const cgroups = await cgroupsOf(appName, containers); const pids = (await Promise.all(cgroups.map(processesUnder))).flat(); const settled = await Promise.all(pids.map(killPendingOrGone)); - return pids.filter((pid, i) => !settled[i]); + const runnable = pids.filter((pid, i) => !settled[i]); + if (runnable.length || containers || !await anyContainerOutsideSlices()) return runnable; + return null; } /** - * Every app with a slice or a container on this node, found without docker or - * the database. + * Every app with a slice or a container record on this node, found without + * docker or the database. + * @param {ContainerRecord[]|null} containers readContainers * @returns {Promise} */ -async function appsOnDisk() { +async function appsOnDisk(containers) { const names = new Set(); const parent = await appSlice.controlGroup('flux.slice'); const slices = parent ? await fs.readdir(parent).catch(() => []) : []; slices.filter((entry) => /^flux-.+\.slice$/.test(entry)) .forEach((entry) => names.add(entry.slice('flux-'.length, -'.slice'.length).replace(/\\x2d/g, '-'))); - (await containersOnDisk()).map(appOf).filter(Boolean).forEach((name) => names.add(name)); + (containers ?? []).map(appOf).filter(Boolean).forEach((name) => names.add(name)); return [...names]; } @@ -218,6 +248,7 @@ function reset() { } module.exports = { + readContainers, containersOf, kill, stillRunnable, diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 12b78780f6..d7eb038938 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -111,11 +111,11 @@ async function removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, reason /** * Ask docker to stop every container of the app, on its normal grace. * @param {string} name + * @param {Array|null} containers appKill.readContainers * @returns {Promise} */ -async function stopApp(name) { - const containers = await appKill.containersOf(name); - await Promise.allSettled(containers.map((container) => dockerService.appDockerStop(container))); +async function stopApp(name, containers) { + await Promise.allSettled(appKill.containersOf(name, containers).map((container) => dockerService.appDockerStop(container))); } /** @@ -131,7 +131,11 @@ async function stopApp(name) { * REMOVAL_DEADLINE_MS; nothing of it runs by then, so the removals take turns. * * The apps are read from the database and from what is on disk, so an app the - * database cannot list is still found. + * database cannot list is still found. Docker's records of its containers are + * read once, within STOP_WINDOW_MS: on a disk that does not answer they are not + * waited for further, every app is still stopped and killed through its slice, + * and an app that may have a container outside its slice is reported with + * runnable null, since what of it can run cannot be known. * @param {Function} installedAppsFn * @param {Function} removeAppLocallyFn */ @@ -141,31 +145,36 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { try { const listed = await withinDeadline(installedAppsFn(), REMOVAL_DEADLINE_MS).catch(() => null); const installed = listed?.status === 'success' ? listed.data.map((app) => app.name) : []; - const names = [...new Set([...installed, ...await appKill.appsOnDisk()])]; + const read = await withinDeadline(appKill.readContainers(), STOP_WINDOW_MS); + const containers = Array.isArray(read) ? read : null; + if (!containers) log.error(`monitorNodeStatus - host stall: docker's container records could not be read (${read === TIMED_OUT ? 'no answer' : 'refused'}); containers outside the app slices cannot be found`); + const names = [...new Set([...installed, ...await appKill.appsOnDisk(containers)])]; log.warn(`monitorNodeStatus - host stall: stopping ${names.length} applications, ${STOP_WINDOW_MS / 1000}s between them`); - await withinDeadline(Promise.allSettled(names.map(stopApp)), STOP_WINDOW_MS); + await withinDeadline(Promise.allSettled(names.map((name) => stopApp(name, containers))), STOP_WINDOW_MS); const stopped = new Map(); // eslint-disable-next-line no-restricted-syntax for (const name of names) { // eslint-disable-next-line no-await-in-loop - let runnable = await appKill.stillRunnable(name); - const killed = runnable.length > 0; + let runnable = await appKill.stillRunnable(name, containers); + const killed = Boolean(runnable?.length); if (killed) { log.error(`monitorNodeStatus - Application ${name} still runs after being asked to stop, killing it`); // eslint-disable-next-line no-await-in-loop - await appKill.kill(name); + await appKill.kill(name, containers); // eslint-disable-next-line no-await-in-loop - runnable = await appKill.stillRunnable(name); - if (runnable.length) { + runnable = await appKill.stillRunnable(name, containers); + if (runnable?.length) { log.error(`monitorNodeStatus - Application ${name} could not be stopped: processes ${runnable.join(', ')} can still run`); - } else { + } else if (runnable) { log.warn(`monitorNodeStatus - Application ${name} can no longer run`); } } + if (runnable === null) log.error(`monitorNodeStatus - Application ${name}: whether it can still run cannot be known`); stopped.set(name, { killed, runnable }); } - // runnable: the processes of each app that can still run its code, empty when none can + // runnable: the processes of each app that can still run its code, empty when + // none can, null when that cannot be known fluxEventBus.publish('hostStall:appsStopped', { apps: names.map((name) => ({ app: name, ...stopped.get(name) })), }); @@ -178,7 +187,7 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { const outcome = await removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) .catch(() => RemovalOutcome.FAILED); const { killed, runnable } = stopped.get(name); - if (!runnable.length && outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + if (runnable?.length === 0 && outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { log.warn(`monitorNodeStatus - Application ${name} was not removed (${outcome}), and can no longer run`); } fluxEventBus.publish('hostStall:appUnloaded', { diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index 94197ace58..f94c3e67d2 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -25,7 +25,8 @@ describe('appKill tests', () => { fs.writeFileSync(path.join(dir, 'cgroup.procs'), pids.map((pid) => `${pid}\n`).join('')); fs.writeFileSync(path.join(dir, 'cgroup.kill'), ''); units.set(path.basename(rel), path.join(SYSTEMD_ROOT, rel)); - if (rel.startsWith('flux.slice/')) units.set('flux.slice', path.join(SYSTEMD_ROOT, 'flux.slice')); + const parent = rel.split('/')[0]; + if (parent !== rel) units.set(parent, path.join(SYSTEMD_ROOT, parent)); return dir; } // `app`: the runonflux.app label FluxOS sets; left out, a container made before it @@ -97,7 +98,7 @@ describe('appKill tests', () => { container(4, 'fluxweb_other', ['14']); slice('flux-my.slice', { [`docker-${id(8)}.scope`]: ['15'] }); - const sent = await appKill.kill('my-app'); + const sent = await appKill.kill('my-app', await appKill.readContainers()); expect(sent).to.have.members([appSliceDir, web, single]); expect(killed).to.have.members([appSliceDir, web, single]); @@ -107,16 +108,23 @@ describe('appKill tests', () => { fs.mkdirSync(path.join(dockerRoot, 'containers', id(5)), { recursive: true }); fs.writeFileSync(path.join(dockerRoot, 'containers', id(5), 'config.v2.json'), JSON.stringify({ Name: '/fluxgone' })); - expect(await appKill.kill('gone')).to.deep.equal([]); + expect(await appKill.kill('gone', await appKill.readContainers())).to.deep.equal([]); expect(killed).to.deep.equal([]); }); + it('kills the app\'s slice alone when docker\'s records could not be read', async () => { + const appSliceDir = slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + container(1, 'fluxweb_app', ['11']); + + expect(await appKill.kill('app', null)).to.deep.equal([appSliceDir]); + }); + it('passes over a container record it cannot read', async () => { const web = container(1, 'fluxweb_app', ['11']); fs.mkdirSync(path.join(dockerRoot, 'containers', id(6)), { recursive: true }); fs.writeFileSync(path.join(dockerRoot, 'containers', id(6), 'config.v2.json'), '{ broken'); - expect(await appKill.kill('app')).to.deep.equal([web]); + expect(await appKill.kill('app', await appKill.readContainers())).to.deep.equal([web]); }); }); @@ -131,23 +139,57 @@ describe('appKill tests', () => { processWith('23', { sigPnd: '0000000000004100' }); // 24 is gone - expect(await appKill.stillRunnable('app')).to.deep.equal(['21']); + expect(await appKill.stillRunnable('app', [])).to.deep.equal(['21']); }); it('does not take another pending signal for the kill', async () => { ['21', '22', '23', '24'].forEach((pid) => processWith(pid, { shdPnd: '0000000000004000' })); - expect(await appKill.stillRunnable('app')).to.deep.equal(['21', '22', '23', '24']); + expect(await appKill.stillRunnable('app', [])).to.deep.equal(['21', '22', '23', '24']); }); it('reads the processes in every scope under the slice, the slice itself holding none', async () => { ['21', '22', '23', '24'].forEach((pid) => processWith(pid)); - expect(await appKill.stillRunnable('app')).to.have.members(['21', '22', '23', '24']); + expect(await appKill.stillRunnable('app', [])).to.have.members(['21', '22', '23', '24']); }); it('names nothing for an app with no cgroup left', async () => { - expect(await appKill.stillRunnable('other')).to.deep.equal([]); + expect(await appKill.stillRunnable('other', [])).to.deep.equal([]); + }); + + describe('when docker\'s records could not be read', () => { + it('names what of the app\'s slice can still run', async () => { + processWith('21'); + container(3, 'fluxother_other', ['31']); + + expect(await appKill.stillRunnable('app', null)).to.deep.equal(['21']); + }); + + it('cannot say the app can no longer run while a container lives outside the slices', async () => { + container(3, 'fluxother_other', ['31']); + + expect(await appKill.stillRunnable('app', null)).to.equal(null); + }); + + it('says the app can no longer run when every container lives in a slice', async () => { + cgroup('system.slice/fluxos.service', ['1']); + + expect(await appKill.stillRunnable('app', null)).to.deep.equal([]); + }); + + it('cannot say the app can no longer run when systemd does not say where system.slice is', async () => { + expect(await appKill.stillRunnable('app', null)).to.equal(null); + }); + }); + }); + + describe('readContainers', () => { + it('answers null when docker\'s records cannot be listed', async () => { + fs.rmSync(path.join(dockerRoot, 'containers'), { recursive: true }); + serviceHelper.runCommand.withArgs('ls').resolves({ stdout: '', error: new Error('No such file or directory') }); + + expect(await appKill.readContainers()).to.equal(null); }); }); @@ -158,14 +200,21 @@ describe('appKill tests', () => { container(2, 'fluxsingle', ['12']); container(3, 'watchtower', ['13']); - expect(await appKill.appsOnDisk()).to.have.members(['sliced-app', 'composed', 'single']); + expect(await appKill.appsOnDisk(await appKill.readContainers())).to.have.members(['sliced-app', 'composed', 'single']); + }); + + it('names every app with a slice when docker\'s records could not be read', async () => { + slice('flux-sliced.slice', {}); + container(1, 'fluxweb_composed', ['11']); + + expect(await appKill.appsOnDisk(null)).to.deep.equal(['sliced']); }); it('names a labelled container\'s app by its label, and a zel component\'s by its name', async () => { record(1, 'fluxgame', 0, { app: 'fluxgame' }); record(2, 'zelcash_legacy'); - expect(await appKill.appsOnDisk()).to.have.members(['fluxgame', 'legacy']); + expect(await appKill.appsOnDisk(await appKill.readContainers())).to.have.members(['fluxgame', 'legacy']); }); }); @@ -176,32 +225,36 @@ describe('appKill tests', () => { record(3, 'fluxweb_composedtoo'); record(4, 'fluxcomposed'); - expect(await appKill.containersOf('composed')).to.have.members(['fluxweb_composed', 'fluxdb_composed', 'fluxcomposed']); + expect(appKill.containersOf('composed', await appKill.readContainers())).to.have.members(['fluxweb_composed', 'fluxdb_composed', 'fluxcomposed']); }); it('takes a container\'s label over its name', async () => { record(1, 'fluxweb_composed', 0, { app: 'other' }); record(2, 'renamed', 0, { app: 'composed' }); - expect(await appKill.containersOf('composed')).to.have.members(['renamed']); + expect(appKill.containersOf('composed', await appKill.readContainers())).to.have.members(['renamed']); }); it('finds a label-less component whose name starts with zel', async () => { record(1, 'zelcash_legacy'); - expect(await appKill.containersOf('legacy')).to.have.members(['zelcash_legacy']); + expect(appKill.containersOf('legacy', await appKill.readContainers())).to.have.members(['zelcash_legacy']); }); it('finds the label-less container of an app whose own name starts with flux', async () => { record(1, 'fluxgame'); - expect(await appKill.containersOf('fluxgame')).to.have.members(['fluxgame']); + expect(appKill.containersOf('fluxgame', await appKill.readContainers())).to.have.members(['fluxgame']); + }); + + it('names none when docker\'s records could not be read', () => { + expect(appKill.containersOf('composed', null)).to.deep.equal([]); }); it('does not take a container FluxOS did not name for an app whose name it ends with', async () => { record(1, 'my_composed'); - expect(await appKill.containersOf('composed')).to.deep.equal([]); + expect(appKill.containersOf('composed', await appKill.readContainers())).to.deep.equal([]); }); }); }); diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 9f03368c15..704c54b9e6 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -159,8 +159,9 @@ describe('nodeStatusMonitor tests', () => { dos = false; bus = { publish: sinon.stub() }; appKill = { + readContainers: sinon.stub().resolves([]), appsOnDisk: sinon.stub().resolves([]), - containersOf: sinon.stub().callsFake(async (name) => [`flux${name}`]), + containersOf: sinon.stub().callsFake((name) => [`flux${name}`]), stillRunnable: sinon.stub().resolves([]), kill: sinon.stub().resolves([]), }; @@ -217,7 +218,7 @@ describe('nodeStatusMonitor tests', () => { }); it('asks docker to stop every container of every app at once', async () => { - appKill.containersOf.withArgs('one').resolves(['fluxweb_one', 'fluxdb_one']); + appKill.containersOf.withArgs('one').returns(['fluxweb_one', 'fluxdb_one']); dockerService.appDockerStop.callsFake(never); const unload = monitor.unloadForHostStall(installed('one', 'two'), sinon.stub().resolves('removed')); @@ -294,7 +295,7 @@ describe('nodeStatusMonitor tests', () => { await monitor.unloadForHostStall(installed('stuck'), sinon.stub().resolves('removed')); - sinon.assert.calledOnceWithExactly(appKill.kill, 'stuck'); + sinon.assert.calledOnceWithExactly(appKill.kill, 'stuck', []); sinon.assert.calledWithMatch(logStub.warn, /stuck can no longer run/); }); @@ -341,6 +342,56 @@ describe('nodeStatusMonitor tests', () => { sinon.assert.calledOnce(remove); }); + it('reads docker\'s records once, and finds, stops, checks and kills every app from that one read', async () => { + const records = [{ id: '1', name: 'fluxa', app: 'a' }]; + appKill.readContainers.resolves(records); + appKill.stillRunnable.onFirstCall().resolves(['41']); + + await monitor.unloadForHostStall(installed('a', 'b'), sinon.stub().resolves('removed')); + + sinon.assert.calledOnce(appKill.readContainers); + sinon.assert.calledWithExactly(appKill.appsOnDisk, records); + ['containersOf', 'stillRunnable'].forEach((call) => appKill[call].getCalls().forEach(({ args }) => expect(args[1], call).to.equal(records))); + sinon.assert.calledOnceWithExactly(appKill.kill, 'a', records); + }); + + it('stops and kills every app through its slice when docker\'s records do not answer within the stop window', async () => { + appKill.readContainers.callsFake(never); + appKill.appsOnDisk.withArgs(null).resolves(['ondisk']); + dockerService.appDockerStop.callsFake(never); + appKill.stillRunnable.callsFake(async (name) => (appKill.kill.calledWith(name) ? [] : ['41'])); + + const unload = monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW * 2); + + sinon.assert.calledWithMatch(logStub.error, /container records could not be read \(no answer\)/); + expect(appKill.kill.getCalls().map(({ args }) => args)).to.deep.equal([['a', null], ['ondisk', null]]); + sinon.assert.calledWithMatch(bus.publish, 'hostStall:appsStopped'); + await unload; + }); + + it('names a refusal to list docker\'s records', async () => { + appKill.readContainers.resolves(null); + + await monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + + sinon.assert.calledWithMatch(logStub.error, /container records could not be read \(refused\)/); + sinon.assert.calledWithExactly(appKill.appsOnDisk, null); + }); + + it('reports an app whose processes cannot all be found as unknown, never as unable to run', async () => { + appKill.stillRunnable.resolves(null); + + await monitor.unloadForHostStall(installed('app'), sinon.stub().resolves('timedOut')); + + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { apps: [{ app: 'app', killed: false, runnable: null }] }); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appUnloaded', { + app: 'app', removal: 'timedOut', killed: false, runnable: null, + }); + sinon.assert.calledWithMatch(logStub.error, /app: whether it can still run cannot be known/); + sinon.assert.neverCalledWithMatch(logStub.warn, /can no longer run/); + }); + it('starts no second removal of an app whose removal is still running, and still checks it', async () => { let finish; const remove = sinon.stub().callsFake(() => new Promise((resolve) => { finish = resolve; })); From 350f59bd9f070996aaf43a926a29c5460f587e63 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:04:48 +0100 Subject: [PATCH 245/369] test(harness): a node's stallable disk can hold every read as well as every write The disk images a suite can freeze (freezableDockerRoot, freezableConfig) are mounted through a device-mapper device of their own. host-stall.stallDisk freezes the filesystem, drops what the kernel caches of it and suspends the device, so a directory listing or a file read on it is held in the kernel the way a disk that stops answering holds it; resumeDisk releases it, and thawEveryNode resumes every held device before anything stops the node. readsHeld is the canary: whether a read on the disk is still held after a few seconds. Co-Authored-By: Claude Opus 5.5 --- test-infra/Dockerfile.fluxos | 1 + test-infra/entrypoint.sh | 29 +++++++++--- test-infra/runner/framework/host-stall.js | 58 +++++++++++++++++++---- 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/test-infra/Dockerfile.fluxos b/test-infra/Dockerfile.fluxos index 742fefb79e..a4b445d0e3 100644 --- a/test-infra/Dockerfile.fluxos +++ b/test-infra/Dockerfile.fluxos @@ -48,6 +48,7 @@ RUN set -eu; \ conntrack \ tcpdump \ e2fsprogs \ + dmsetup \ systemd \ dbus \ openssh-client \ diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 3b4c067457..80bf7c88f7 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -52,15 +52,32 @@ mkdir -p /dat/var/lib/fluxd \ /dat/usr/lib/fluxwatchdog \ /mnt/appdata/flux-apps -# A node whose docker data a suite freezes (framework/host-stall.js) keeps it on a -# filesystem of its own, mounted before docker starts. Freezing it holds every write -# docker makes there in the kernel, which is what a stalled host does to docker. The -# image lives on the node's volume, so a restarted node mounts the same data again. +# Mount a filesystem image through a device-mapper device of its own, so a suite +# can freeze it (every write held) or suspend the device (every read and write +# held), framework/host-stall.js. Device-mapper names are the host kernel's, so +# each carries the node's hostname. The device and its loop are released lazily: +# both go away with the last mount of them, when the node's container does. +mount_stallable() { + local img="$1" mnt="$2" name loop + mountpoint -q "$mnt" && return 0 + name="flux-$(hostname)-$(basename "$mnt")" + loop="$(losetup -f --show "$img")" + dmsetup create --noudevsync "$name" --table "0 $(blockdev --getsz "$loop") linear $loop 0" + dmsetup mknodes "$name" + mount "/dev/mapper/$name" "$mnt" + dmsetup remove --deferred "$name" + losetup -d "$loop" +} + +# A node whose docker data a suite stalls (framework/host-stall.js) keeps it on a +# filesystem of its own, mounted before docker starts. Holding it holds docker's +# threads in the kernel, which is what a stalled host does to docker. The image +# lives on the node's volume, so a restarted node mounts the same data again. if [ "$FLUX_FREEZABLE_DOCKER_ROOT" = "true" ]; then img=/mnt/appdata/docker-root.img [ -f "$img" ] || { truncate -s 8G "$img" && mkfs.ext4 -q -F "$img"; } mkdir -p /mnt/appdata/docker - mountpoint -q /mnt/appdata/docker || mount -o loop "$img" /mnt/appdata/docker + mount_stallable "$img" /mnt/appdata/docker fi # The same for the node's FluxOS config directory, where FluxOS keeps the record of @@ -76,7 +93,7 @@ if [ "$FLUX_FREEZABLE_CONFIG" = "true" ]; then mount -o loop "$img" "$seed" && cp -a "$cfg/." "$seed/" && chown "$owner" "$seed" && umount "$seed" rmdir "$seed" fi - mountpoint -q "$cfg" || mount -o loop "$img" "$cfg" + mount_stallable "$img" "$cfg" fi # In stub mode FluxOS only needs somewhere to read an API key from; the calls diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index 880dccefbd..07defa802a 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -1,11 +1,14 @@ // A stalled host, on a real kernel: the filesystem a node's docker keeps its data on, -// frozen. Every write docker makes there is then held in the kernel - its threads sit -// in uninterruptible sleep and never run - until the filesystem is thawed, which is -// what a host that stops returning the disk does to docker. +// held. Frozen (freezeDocker), every write docker makes there is held in the kernel - +// its threads sit in uninterruptible sleep and never run - until the filesystem is +// thawed, which is what a host that stops returning the disk does to docker. Stalled +// (stallDisk), every read is held as well, which is what a disk that stops answering +// does to everything on it. // -// The node is booted with createTestEnv({ freezableDockerRoot: [index] }), which -// mounts its docker data root on a filesystem of its own. The env's teardown thaws -// it, so a node a failing test left frozen can still be stopped. +// The node is booted with createTestEnv({ freezableDockerRoot: [index] }), or +// freezableConfig, which mounts that directory on a filesystem of its own, through a +// device-mapper device of its own. The env's teardown thaws and resumes it, so a node +// a failing test left held can still be stopped. import { execInContainer } from './container.js'; export const DOCKER_ROOT = '/mnt/appdata/docker'; @@ -44,15 +47,54 @@ export async function thawConfig(container) { await run(container, `out=$(fsfreeze -u ${FLUXOS_CONFIG} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }`, 'thawConfig'); } +// The device-mapper device a stallable mount sits on, in the node's shell. +const deviceOf = (mountpoint) => `$(findmnt -no SOURCE ${mountpoint} | sed -n 's|^/dev/mapper/||p')`; + +/** + * Hold every read and write to the filesystem at `mountpoint`, until resumeDisk: + * the filesystem is frozen, so a write is held before it reaches the cache, and + * its device suspended. What of it the kernel has cached is dropped in between, + * for that filesystem only, so a read of a file or a directory listing on it + * reaches the device and is held. + * @param {object} container The node. + * @param {string} mountpoint DOCKER_ROOT or FLUXOS_CONFIG, mounted stallable. + */ +export async function stallDisk(container, mountpoint) { + await run(container, `d=${deviceOf(mountpoint)}; [ -n "$d" ]`, `${mountpoint} is not on a device of its own: boot the node with freezableDockerRoot or freezableConfig`); + await run(container, `fsfreeze -f ${mountpoint} && find ${mountpoint} -xdev -type f -print0 | xargs -0 -r -n 64 -P 8 sh -c 'for f; do dd if="$f" iflag=nocache count=0 status=none; done' sh` + + ` && blockdev --flushbufs $(findmnt -no SOURCE ${mountpoint}) && dmsetup suspend --nolockfs --noflush ${deviceOf(mountpoint)}`, `stallDisk ${mountpoint}`); +} + +/** Let reads and writes reach the filesystem at `mountpoint` again. One not stalled is left as it is. */ +export async function resumeDisk(container, mountpoint) { + // the thaw writes to the device, so the device is resumed first + await run(container, `dmsetup resume ${deviceOf(mountpoint)} && { out=$(fsfreeze -u ${mountpoint} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }; }`, `resumeDisk ${mountpoint}`); +} + +/** + * Whether a listing of `dir` and a read of every file named `file` below it are + * still held after `overMs`: run in the background and left to finish once the + * disk is resumed, since a read held by the kernel cannot be killed. + * @returns {Promise} + */ +export async function readsHeld(container, dir, { file, overMs = 3000 } = {}) { + const marker = `/tmp/reads-returned-${Date.now()}`; + const read = `ls ${dir} >/dev/null${file ? ` && find ${dir} -name ${file} -exec cat {} + >/dev/null` : ''}`; + const result = await execInContainer(container, + `setsid sh -c '${read} && touch ${marker}' /dev/null 2>&1 & sleep ${overMs / 1000}; test ! -e ${marker}`); + return result.exitCode === 0; +} + /** - * Thaw every node's docker data root and config directory that is frozen, and every cgroup frozen in it + * Thaw and resume every node's docker data root and config directory that is held, and every cgroup frozen in it * (systemd-control.freezeUnit), and pass over the rest: the last step before * anything reads a node through its docker or stops it, since a node whose docker * is held can be neither read nor stopped. Never rejects. * @param {Array<{container?: object}>} clients */ export async function thawEveryNode(clients) { - const thaw = `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; fsfreeze -u ${FLUXOS_CONFIG} 2>/dev/null; ` + const thaw = `for m in ${DOCKER_ROOT} ${FLUXOS_CONFIG}; do d=${deviceOf('$m')}; [ -n "$d" ] && dmsetup resume "$d" 2>/dev/null; done; ` + + `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; fsfreeze -u ${FLUXOS_CONFIG} 2>/dev/null; ` + 'for f in $(find /sys/fs/cgroup -name cgroup.freeze 2>/dev/null); do [ "$(cat $f)" = 1 ] && echo 0 > $f; done; true'; await Promise.all(clients.map(async (client) => { if (!client?.container) return; From 2c508ed0b061aee9bef6dc1903ac8a6ea4b0cb1d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:04:48 +0100 Subject: [PATCH 246/369] test(harness): 3303 stops a node's apps when docker's disk stops answering reads A node whose docker disk holds every read cannot read docker's records of its containers. Its app is still stopped within the record read's deadline and one stop window, and reported as unable to run, every container on the node being in its slice. Co-Authored-By: Claude Opus 5.5 --- ...lled-host-takes-the-node-out-of-service.js | 73 ++++++++++++++++++- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 683ca91795..a7848430e2 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -9,8 +9,9 @@ import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { buildSeedableTestApp } from '../framework/seed-helper.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { - freezeDocker, thawDocker, freezeConfig, thawConfig, FLUXOS_CONFIG, + freezeDocker, thawDocker, freezeConfig, thawConfig, FLUXOS_CONFIG, DOCKER_ROOT, thawEveryNode, stuckThreads, processesThatCanRun, containerCgroup, + stallDisk, resumeDisk, readsHeld, } from '../framework/host-stall.js'; import { DOCKERD_UNIT, freezeUnit, restartFluxos, unitCgroup, @@ -33,7 +34,9 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // apps has every one of them stopped within that one window, long before even one // removal's deadline. A node whose FluxOS config directory, where the trip is // recorded, is frozen as well stops its apps all the same, and records the trip -// once the directory takes writes again. +// once the directory takes writes again. A node whose docker disk stops answering +// reads as well, so docker's own records of its containers cannot be read, stops +// and kills its apps through their slices within the same window. // // The nodes run systemd, so docker uses its systemd cgroup driver and every app's // containers live in the app's own slice, as on a production node. The node the @@ -53,6 +56,8 @@ const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; const OWNER = 'hostStall'; const ARCANE = [0, 1, 2]; const LEGACY_UNPRIVILEGED = 3; +// an Arcane node of its own: every case leaves its node out of service +const DISK_STALLED = 4; const byOwner = (data) => data.owner === OWNER; const outOfServiceForTheStall = (data) => data.dosState >= 100 @@ -92,9 +97,9 @@ describe('a stalled host takes the node out of service', function () { this.timeout(480000); env = await createTestEnv({ hookCtx: this, - nodes: 4, + nodes: 5, tickerAutostart: false, - freezableDockerRoot: [0, 1, 2, 3], + freezableDockerRoot: [0, 1, 2, 3, DISK_STALLED], freezableConfig: [0, 1, 2, 3], legacyNodes: [LEGACY_UNPRIVILEGED], unprivilegedNodes: [LEGACY_UNPRIVILEGED], @@ -400,4 +405,64 @@ describe('a stalled host takes the node out of service', function () { }); }); }); + + describe('a stall past the threshold, on a node whose docker disk stops answering reads', function () { + const diskName = `e2ehoststalldisk${Date.now()}`; + let diskClient; + let diskContainer; + let anchor; + let readsWereHeld; + let runningBefore; + let trippedAt; + let stopped; + let runnableAfterWindow; + + before(async function () { + this.timeout(480000); + diskClient = env.clients[DISK_STALLED]; + ({ container: diskContainer } = diskClient); + await installOn(env, diskName, [DISK_STALLED]); + await waitForUp(diskClient, diskName, `${diskName} is up`); + anchor = diskClient.getLastEventId(); + step(`stalling node ${DISK_STALLED}'s docker disk`); + await stallDisk(diskContainer, DOCKER_ROOT); + readsWereHeld = await readsHeld(diskContainer, `${DOCKER_ROOT}/containers`, { file: 'config.v2.json' }); + runningBefore = await sliceProcessesThatCanRun(diskContainer, diskName); + const trip = await diskClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + // the records' read waits one window, and the stop another + stopped = (await diskClient.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS * 2 + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + runnableAfterWindow = trippedAt ? await sliceProcessesThatCanRun(diskContainer, diskName) : null; + step('read the app after the stop window'); + }); + + after(async function () { + if (!diskClient) return; + await resumeDisk(diskContainer, DOCKER_ROOT).catch(() => {}); + await thawEveryNode([diskClient]); + }); + + it('held every read of docker\'s records of its containers, while the app could run', function () { + expect(readsWereHeld, 'a read of docker\'s container records returned, so the disk was never held').to.equal(true); + expect(runningBefore, 'the app was not running, so there was nothing to stop').to.not.be.empty; + }); + + it('takes the node out of service', function () { + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('stops the app though docker\'s records cannot be read', function () { + expect(STOP_WINDOW_MS * 2 + KILL_SLACK_MS, 'fixture: the record read, the window and the kill do not fit inside one removal\'s deadline').to.be.below(REMOVAL_DEADLINE_MS); + expect(stopped, 'the node never stopped its apps while docker\'s records could not be read').to.not.equal(null); + expect(runnableAfterWindow, `${diskName} can still run after the stop window`).to.deep.equal([]); + }); + + it('says nothing of the app can run, every container on the node being in its slice', function () { + const app = stopped?.apps.find((a) => a.app === diskName); + expect(app, 'the node did not name the app among those it stopped').to.not.equal(undefined); + expect(app.runnable, 'the node reports the app as one it cannot tell can still run, or names its processes').to.deep.equal([]); + }); + }); }); From 8d59d38efeee2642026391c3ae32364e33f5eb51 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:41:10 +0100 Subject: [PATCH 247/369] fix(apps): a host stall's trip and its count of readings are kept on /dev/shm The trip, and every stuck thread's count of readings toward it, are kept in /dev/shm/flux-host-stall.json after each reading and restored at start. The tmpfs is memory, so a start reads it back whatever disk has stalled, and a reboot empties it, which is when a trip ends. A FluxOS restart mid-stall therefore neither forgets a trip nor starts the count again. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 2 - ZelBack/config/default.js | 3 - ZelBack/src/services/hostStallService.js | 94 ++++++---------- tests/unit/globalconfig/default.js | 3 - tests/unit/hostStallService.test.js | 137 ++++++++--------------- 5 files changed, 81 insertions(+), 158 deletions(-) diff --git a/.gitignore b/.gitignore index 8e3a4c2ec4..863ad99093 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,6 @@ yarn-error.log* /test/e2e/reports/ selenium-debug.log config/userconfig.js -config/hostStall.json -config/hostStall.json.tmp ZelApps appvolumes .nyc_output/ diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index dd21e70900..df3e05bc7e 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -50,9 +50,6 @@ module.exports = { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, stopWindowMs: 60 * 1000, - // How long a start waits to read the trip record. A healthy disk reads it in - // milliseconds. - recordReadMs: 10 * 1000, }, database: { url: '127.0.0.1', diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 404ec9586a..0b7266a944 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -12,9 +12,10 @@ * takes no readings while it is paused, so a pause cannot read as a stall. * * Out of service lasts until the machine reboots, through any number of FluxOS - * restarts: the trip is recorded with the kernel's boot id, and a start in the - * same boot restores it. The record is a file, not a database row, because the - * database may be what is stuck. + * restarts, and so does the count of readings toward it. Both are kept in a file + * on /dev/shm, the tmpfs every Linux mounts at boot: memory, so reading it cannot + * stall with a disk, and emptied by a reboot and by nothing else. Not a database + * row, because the database may be what is stuck. * * WHAT IS READ. The threads of the processes FluxOS depends on, in FluxOS's own * PID namespace. An app container that runs one of the same programs has its @@ -27,13 +28,10 @@ const path = require('path'); const config = require('config'); const log = require('../lib/log'); const fluxNetworkHelper = require('./fluxNetworkHelper'); -const { fluxDirPath } = require('./utils/appConstants'); -const { withinDeadline, TIMED_OUT } = require('./utils/withinDeadline'); const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); const READ_INTERVAL_MS = config.hostStall.readIntervalMs; const STUCK_INTERVALS = config.hostStall.stuckIntervals; -const RECORD_READ_MS = config.hostStall.recordReadMs; /** * Every dependency thread in D at the last reading, keyed by process, thread @@ -41,6 +39,8 @@ const RECORD_READ_MS = config.hostStall.recordReadMs; * @type {Map} */ let tracked = new Map(); +/** Why the node is out of service for a host stall, once it is. */ +let tripReason = null; let readTimer = null; // Advanced by stop(), so a reading still under way when the service stops // neither keeps what it read nor acts on it. @@ -49,14 +49,13 @@ let generation = 0; const tripListeners = []; const DEFAULT_PATHS = { procRoot: '/proc', - recordFile: path.join(fluxDirPath, 'config', 'hostStall.json'), - bootIdPath: config.system.bootIdPath, + stateFile: '/dev/shm/flux-host-stall.json', }; let paths = { ...DEFAULT_PATHS }; /** - * @param {{procRoot?: string, recordFile?: string, bootIdPath?: string}} options - * Where /proc, the trip record and the boot id are; each one given replaces it. + * @param {{procRoot?: string, stateFile?: string}} options + * Where /proc and the kept state are; each one given replaces it. */ function usePaths(options) { paths = { ...paths, ...options }; @@ -166,29 +165,17 @@ function describe(thread) { } /** - * @returns {Promise} This boot's id, or null when it cannot be read. + * Keep the trip and every thread's count, so a FluxOS restart before the next + * reboot carries both on. */ -async function currentBootId() { - const id = (await readOrNull(paths.bootIdPath))?.trim(); - return id || null; -} - -/** - * Record the trip, so a FluxOS restart in this boot restores it. - * @param {string} reason - */ -async function recordTrip(reason) { +async function keepState() { + const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; try { - const bootId = await currentBootId(); - if (!bootId) { - log.warn('hostStall - the boot id cannot be read; the trip lasts only as long as this process'); - return; - } - const staged = `${paths.recordFile}.tmp`; - await fs.writeFile(staged, JSON.stringify({ bootId, reason })); - await fs.rename(staged, paths.recordFile); + const staged = `${paths.stateFile}.tmp`; + await fs.writeFile(staged, JSON.stringify({ trip, threads: [...tracked] })); + await fs.rename(staged, paths.stateFile); } catch (error) { - log.error(`hostStall - the trip could not be recorded: ${error.message}`); + log.error(`hostStall - the state could not be kept: ${error.message}`); } } @@ -208,23 +195,20 @@ function onTrip(listener) { if (fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) listener(); } -/** Restore a trip recorded in this boot, and discard one from an earlier boot. */ -async function restoreTrip() { +/** Restore what an earlier FluxOS in this boot kept: the trip, and every thread's count. */ +async function restoreState() { + const text = await readOrNull(paths.stateFile); + if (text === null) return; try { - const text = await readOrNull(paths.recordFile); - if (text === null) return; - const bootId = await currentBootId(); - if (!bootId) return; - const { bootId: recorded, reason } = JSON.parse(text); - if (recorded !== bootId) { - await fs.rm(paths.recordFile, { force: true }); - log.info('hostStall - the recorded trip is from an earlier boot and is discarded'); - return; + const { trip, threads } = JSON.parse(text); + tracked = new Map(threads); + if (trip) { + tripReason = trip; + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, trip); + announceTrip(); } - fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); - announceTrip(); } catch (error) { - log.error(`hostStall - the recorded trip could not be read: ${error.message}`); + log.error(`hostStall - the kept state could not be read: ${error.message}`); } } @@ -237,12 +221,9 @@ async function act() { } fluxNetworkHelper.setPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL, describe(longest)); if (longest.intervals >= STUCK_INTERVALS && !fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { - const reason = `Host stall: ${describe(longest)}`; - fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, reason); - // Announced before it is recorded, and the reading does not wait for the record: - // the disk the record goes to may be the one that stalled. + tripReason = `Host stall: ${describe(longest)}`; + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, tripReason); announceTrip(); - recordTrip(reason); } } @@ -257,6 +238,7 @@ async function readOnce(options = {}) { if (startedIn !== generation) return; tracked = carryForward(threads); await act(); + await keepState(); } async function readAndReschedule() { @@ -269,23 +251,16 @@ async function readAndReschedule() { } /** - * Restore a trip recorded in this boot, then read now, and every + * Restore what an earlier FluxOS in this boot kept, then read now, and every * READ_INTERVAL_MS after each reading ends. - * - * A record that cannot be read within RECORD_READ_MS is not waited for further: - * its disk is stalled, and a stall still under way takes the node out of service - * again at the readings. A restore that ends later still restores. * @param {object} [options] Paths, as usePaths takes them. - * @returns {Promise} Resolves once a recorded trip is restored, or the - * record could not be read within RECORD_READ_MS. + * @returns {Promise} Resolves once the kept state is restored. */ async function start(options = {}) { if (readTimer !== null) return; usePaths(options); + await restoreState(); readTimer = setTimeout(readAndReschedule, 0); - if (await withinDeadline(restoreTrip(), RECORD_READ_MS) === TIMED_OUT) { - log.error(`hostStall - the trip record could not be read within ${RECORD_READ_MS / 1000}s; going on without it`); - } } /** Take no more readings and forget every thread. */ @@ -294,6 +269,7 @@ function stop() { clearTimeout(readTimer); readTimer = null; tracked = new Map(); + tripReason = null; paths = { ...DEFAULT_PATHS }; tripListeners.length = 0; } diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 7482ab75bb..bc7a836e61 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -33,9 +33,6 @@ module.exports = { readIntervalMs: 5 * 60 * 1000, stuckIntervals: 6, stopWindowMs: 60 * 1000, - // How long a start waits to read the trip record. A healthy disk reads it in - // milliseconds. - recordReadMs: 10 * 1000, }, database: { url: database, diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 3add177cb9..343d4022ee 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -42,8 +42,7 @@ describe('hostStallService tests', () => { const stuckMongod = (switches = 40) => processWith('100', 'mongod', [{ tid: '101', state: 'D', switches }, { tid: '102', state: 'S', switches: 5 }]); const read = () => hostStallService.readOnce(paths); - const bootIs = (id) => fs.writeFileSync(paths.bootIdPath, `${id}\n`); - const record = () => JSON.parse(fs.readFileSync(paths.recordFile, 'utf8')); + const kept = () => JSON.parse(fs.readFileSync(paths.stateFile, 'utf8')); async function readTimes(n) { for (let i = 0; i < n; i += 1) await read(); // eslint-disable-line no-await-in-loop } @@ -56,8 +55,7 @@ describe('hostStallService tests', () => { fs.mkdirSync(path.join(procRoot, 'self', 'ns'), { recursive: true }); fs.symlinkSync('pid:[4026531836]', path.join(procRoot, 'self', 'ns', 'pid')); fs.writeFileSync(path.join(procRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); - paths = { procRoot, recordFile: path.join(procRoot, 'hostStall.json'), bootIdPath: path.join(procRoot, 'boot_id') }; - bootIs('boot-one'); + paths = { procRoot, stateFile: path.join(procRoot, 'flux-host-stall.json') }; }); afterEach(() => { @@ -224,10 +222,9 @@ describe('hostStallService tests', () => { sinon.assert.calledOnce(listener); }); - it('tells a listener when a trip recorded in this boot is restored at start', async () => { + it('tells a listener when a kept trip is restored at start', async () => { stuckMongod(); await readTimes(stuckIntervals + 1); - while (!fs.existsSync(paths.recordFile)) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop hostStallService.stop(); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); const listener = sinon.stub(); @@ -239,37 +236,32 @@ describe('hostStallService tests', () => { }); }); - describe('the trip record', () => { - const settled = () => new Promise((resolve) => { setImmediate(resolve); }); - // The trip, and its record on disk where the boot id says one is made. - async function trip() { - stuckMongod(); - await readTimes(stuckIntervals + 1); - while (fs.existsSync(paths.bootIdPath) && !fs.existsSync(paths.recordFile)) await settled(); // eslint-disable-line no-await-in-loop - } - // Every write of the record to disk, held until released. - function holdRecordWrites() { - const writeFile = fs.promises.writeFile.bind(fs.promises); - const held = []; - sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.recordFile) - ? new Promise((resolve) => { held.push(() => resolve(writeFile(file, ...rest))); }) - : writeFile(file, ...rest))); - return held; - } + describe('the kept state', () => { async function restart() { hostStallService.stop(); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); await hostStallService.start(paths); } - it('records the trip with this boot\'s id', async () => { - await trip(); + it('keeps the trip and every thread\'s count after each reading', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + + const { trip, threads } = kept(); + expect(trip).to.equal(fluxNetworkHelper.getDosMessage()); + expect(threads.map(([, thread]) => [thread.tid, thread.intervals])).to.deep.equal([['101', stuckIntervals]]); + }); + + it('keeps no trip before the node goes out of service', async () => { + stuckMongod(); + await readTimes(stuckIntervals); - expect(record()).to.deep.equal({ bootId: 'boot-one', reason: fluxNetworkHelper.getDosMessage() }); + expect(kept().trip).to.equal(null); }); - it('restores the trip at a start in the same boot', async () => { - await trip(); + it('restores the trip at a start', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); const reason = fluxNetworkHelper.getDosMessage(); await restart(); @@ -278,37 +270,47 @@ describe('hostStallService tests', () => { expect(fluxNetworkHelper.getDosMessage()).to.equal(reason); }); - it('discards a trip from an earlier boot, and stays in service', async () => { - await trip(); - bootIs('boot-two'); + it('carries a stuck thread\'s count through a restart', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + await restart(); + expect(outOfService(), 'out of service before the readings reached the threshold').to.equal(false); + + await read(); + + expect(outOfService(), 'the restart started the count again').to.equal(true); + }); + it('starts the count again for a thread that ran while FluxOS was not running', async () => { + stuckMongod(); + await readTimes(stuckIntervals); await restart(); + stuckMongod(41); + + await read(); expect(outOfService()).to.equal(false); - expect(fs.existsSync(paths.recordFile)).to.equal(false); }); - it('records nothing without a boot id, and is still out of service', async () => { - fs.rmSync(paths.bootIdPath); - - await trip(); + it('starts in service with nothing kept', async () => { + await restart(); - expect(outOfService()).to.equal(true); - expect(fs.existsSync(paths.recordFile)).to.equal(false); + expect(outOfService()).to.equal(false); }); - it('restores nothing and discards nothing at a start without a boot id', async () => { - await trip(); - fs.rmSync(paths.bootIdPath); + it('starts in service when the kept state cannot be read', async () => { + fs.writeFileSync(paths.stateFile, '{ not json'); await restart(); expect(outOfService()).to.equal(false); - expect(fs.existsSync(paths.recordFile)).to.equal(true); }); - it('takes the node out of service and tells its listeners while the record cannot be written', async () => { - holdRecordWrites(); + it('takes the node out of service and tells its listeners when the state cannot be kept', async () => { + const writeFile = fs.promises.writeFile.bind(fs.promises); + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.stateFile) + ? Promise.reject(new Error('read-only')) + : writeFile(file, ...rest))); const listener = sinon.stub(); hostStallService.onTrip(listener); stuckMongod(); @@ -318,53 +320,6 @@ describe('hostStallService tests', () => { expect(outOfService()).to.equal(true); sinon.assert.calledOnce(listener); }); - - it('goes on reading while the record cannot be written', async () => { - holdRecordWrites(); - stuckMongod(); - await readTimes(stuckIntervals + 1); - processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 41 }]); - - const outcome = await Promise.race([read().then(() => 'read'), new Promise((resolve) => { setTimeout(() => resolve('waiting'), 1000); })]); - - expect(outcome, 'a reading waited on the record').to.equal('read'); - expect(placementHeld(), 'the reading after the trip released the hold').to.equal(false); - }); - - it('starts without the record when it cannot be read in time, and restores it when the read ends', async () => { - await trip(); - hostStallService.stop(); - fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); - const readFile = fs.promises.readFile.bind(fs.promises); - let finishRead = null; - sinon.stub(fs.promises, 'readFile').callsFake((file, ...rest) => (file === paths.recordFile - ? new Promise((resolve) => { finishRead = () => resolve(readFile(file, ...rest)); }) - : readFile(file, ...rest))); - const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); - try { - const started = hostStallService.start(paths); - let done = false; - started.then(() => { done = true; }); - await clock.tickAsync(config.hostStall.recordReadMs - 1); - expect(done, 'the start returned before its wait for the record ran out').to.equal(false); - await clock.tickAsync(1); - await started; - expect(outOfService(), 'restored without a record').to.equal(false); - - finishRead(); - while (!outOfService()) await clock.tickAsync(0); // eslint-disable-line no-await-in-loop - } finally { - clock.restore(); - } - }); - - it('starts in service when the record cannot be read', async () => { - fs.writeFileSync(paths.recordFile, '{ not json'); - - await restart(); - - expect(outOfService()).to.equal(false); - }); }); it('names no wait point the kernel does not show', async () => { From 06f64bea9ad97b6f81b2c0d943277204cd1425f1 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:41:10 +0100 Subject: [PATCH 248/369] test(harness): 3303 restarts FluxOS mid-stall and the node still goes out of service on time The node restarts FluxOS two readings before it would go out of service and reads every 20 s, so the readings a restart would lose take far longer than the restart; it must go out of service within three readings of the restart. The node's FluxOS config directory is no longer mounted on a disk of its own: nothing a stall depends on is kept there. Co-Authored-By: Claude Opus 5.5 --- test-infra/entrypoint.sh | 16 --- test-infra/runner/framework/host-stall.js | 32 ++---- test-infra/runner/framework/test-env.js | 12 +-- ...lled-host-takes-the-node-out-of-service.js | 100 ++++++++---------- 4 files changed, 58 insertions(+), 102 deletions(-) diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 80bf7c88f7..96b8a9c511 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -80,22 +80,6 @@ if [ "$FLUX_FREEZABLE_DOCKER_ROOT" = "true" ]; then mount_stallable "$img" /mnt/appdata/docker fi -# The same for the node's FluxOS config directory, where FluxOS keeps the record of -# a host stall: frozen, every write FluxOS makes there is held in the kernel. Seeded -# with the directory's own files and owner on first boot. -if [ "$FLUX_FREEZABLE_CONFIG" = "true" ]; then - cfg="${FLUXOS_PATH:-/flux}/config" - img=/mnt/appdata/flux-config.img - owner="$(stat -c %u:%g "$cfg")" - if [ ! -f "$img" ]; then - truncate -s 64M "$img" && mkfs.ext4 -q -F "$img" - seed="$(mktemp -d)" - mount -o loop "$img" "$seed" && cp -a "$cfg/." "$seed/" && chown "$owner" "$seed" && umount "$seed" - rmdir "$seed" - fi - mount_stallable "$img" "$cfg" -fi - # In stub mode FluxOS only needs somewhere to read an API key from; the calls # themselves go to the shared stub. In binary mode the config is syncthing's own # and this fixture must not be in the way of it. diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js index 07defa802a..9a45b82e70 100644 --- a/test-infra/runner/framework/host-stall.js +++ b/test-infra/runner/framework/host-stall.js @@ -5,15 +5,13 @@ // (stallDisk), every read is held as well, which is what a disk that stops answering // does to everything on it. // -// The node is booted with createTestEnv({ freezableDockerRoot: [index] }), or -// freezableConfig, which mounts that directory on a filesystem of its own, through a -// device-mapper device of its own. The env's teardown thaws and resumes it, so a node +// The node is booted with createTestEnv({ freezableDockerRoot: [index] }), which +// mounts its docker data root on a filesystem of its own, through a device-mapper +// device of its own. The env's teardown thaws and resumes it, so a node // a failing test left held can still be stopped. import { execInContainer } from './container.js'; export const DOCKER_ROOT = '/mnt/appdata/docker'; -// A harness node's FluxOS directory is /flux, legacy and Arcane alike. -export const FLUXOS_CONFIG = '/flux/config'; async function run(container, command, what) { const result = await execInContainer(container, command); @@ -33,20 +31,6 @@ export async function thawDocker(container) { await run(container, `out=$(fsfreeze -u ${DOCKER_ROOT} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }`, 'thawDocker'); } -/** - * Hold every write the node's FluxOS makes to its config directory, until - * thawConfig. The node is booted with createTestEnv({ freezableConfig: [index] }). - */ -export async function freezeConfig(container) { - await run(container, `mountpoint -q ${FLUXOS_CONFIG}`, `${FLUXOS_CONFIG} is not a mount of its own: boot the node with freezableConfig`); - await run(container, `fsfreeze -f ${FLUXOS_CONFIG}`, 'freezeConfig'); -} - -/** Let the node's FluxOS write its config directory again. One not frozen is left as it is. */ -export async function thawConfig(container) { - await run(container, `out=$(fsfreeze -u ${FLUXOS_CONFIG} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }`, 'thawConfig'); -} - // The device-mapper device a stallable mount sits on, in the node's shell. const deviceOf = (mountpoint) => `$(findmnt -no SOURCE ${mountpoint} | sed -n 's|^/dev/mapper/||p')`; @@ -57,10 +41,10 @@ const deviceOf = (mountpoint) => `$(findmnt -no SOURCE ${mountpoint} | sed -n 's * for that filesystem only, so a read of a file or a directory listing on it * reaches the device and is held. * @param {object} container The node. - * @param {string} mountpoint DOCKER_ROOT or FLUXOS_CONFIG, mounted stallable. + * @param {string} mountpoint DOCKER_ROOT, mounted stallable. */ export async function stallDisk(container, mountpoint) { - await run(container, `d=${deviceOf(mountpoint)}; [ -n "$d" ]`, `${mountpoint} is not on a device of its own: boot the node with freezableDockerRoot or freezableConfig`); + await run(container, `d=${deviceOf(mountpoint)}; [ -n "$d" ]`, `${mountpoint} is not on a device of its own: boot the node with freezableDockerRoot`); await run(container, `fsfreeze -f ${mountpoint} && find ${mountpoint} -xdev -type f -print0 | xargs -0 -r -n 64 -P 8 sh -c 'for f; do dd if="$f" iflag=nocache count=0 status=none; done' sh` + ` && blockdev --flushbufs $(findmnt -no SOURCE ${mountpoint}) && dmsetup suspend --nolockfs --noflush ${deviceOf(mountpoint)}`, `stallDisk ${mountpoint}`); } @@ -86,15 +70,15 @@ export async function readsHeld(container, dir, { file, overMs = 3000 } = {}) { } /** - * Thaw and resume every node's docker data root and config directory that is held, and every cgroup frozen in it + * Thaw and resume every node's docker data root that is held, and every cgroup frozen in it * (systemd-control.freezeUnit), and pass over the rest: the last step before * anything reads a node through its docker or stops it, since a node whose docker * is held can be neither read nor stopped. Never rejects. * @param {Array<{container?: object}>} clients */ export async function thawEveryNode(clients) { - const thaw = `for m in ${DOCKER_ROOT} ${FLUXOS_CONFIG}; do d=${deviceOf('$m')}; [ -n "$d" ] && dmsetup resume "$d" 2>/dev/null; done; ` - + `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; fsfreeze -u ${FLUXOS_CONFIG} 2>/dev/null; ` + const thaw = `d=${deviceOf(DOCKER_ROOT)}; [ -n "$d" ] && dmsetup resume "$d" 2>/dev/null; ` + + `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; ` + 'for f in $(find /sys/fs/cgroup -name cgroup.freeze 2>/dev/null); do [ "$(cat $f)" = 1 ] && echo 0 > $f; done; true'; await Promise.all(clients.map(async (client) => { if (!client?.container) return; diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index f1f4f5727c..5bcfbb20ef 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -971,7 +971,7 @@ export async function createTestEnv({ tickerAutostart = false, discoveryAutostart = false, nodeStatusOverrides = {}, rpcFailures = [], bootContext = 'running', initialHeight = DEFAULT_INITIAL_HEIGHT, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation = {}, locationTable = null, staticIp = true, policy = null, policySeeds = null, - awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], freezableConfig = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], + awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], // Firewalled nodes boot with ufw active as their install leaves it - the legacy // installer's rules on a legacy node, the ISO's on an Arcane one - and FluxOS // adds its own on top. true for every node, or a list of node indices. Off @@ -988,11 +988,6 @@ export async function createTestEnv({ for (const index of freezableDockerRoot) { if (!Number.isInteger(index) || index < 0 || index >= nodes) throw new Error(`createTestEnv: freezableDockerRoot names node ${index}, outside a fleet of ${nodes}`); } - // Nodes whose FluxOS config directory is a filesystem a suite can freeze - // (framework/host-stall.js), by node index. - for (const index of freezableConfig) { - if (!Number.isInteger(index) || index < 0 || index >= nodes) throw new Error(`createTestEnv: freezableConfig names node ${index}, outside a fleet of ${nodes}`); - } for (const index of pausedSyncthingNodes) { if (syncthing !== 'binary') throw new Error('createTestEnv: pausedSyncthingNodes needs syncthing: \'binary\''); if (legacyNodes.includes(index)) throw new Error(`createTestEnv: paused syncthing node ${index} is legacy, whose syncthing FluxOS starts`); @@ -1283,7 +1278,7 @@ export async function createTestEnv({ // mongo starts, i.e. inside the fleet boot, where the waits at risk are the // boot's own. await startInfraDeathWatch(env); - await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot, freezableConfig); + await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot); return env; } catch (err) { // Boot failed: the env owns everything started so far. The shared teardown @@ -1312,7 +1307,7 @@ function mergeConfigs(base, override) { return result; } -async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], freezableConfig = []) { +async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = []) { // Everything built here registers onto the env shell as it comes up, so a // boot-phase throw leaves the partial state reachable (see makeEnvShell). const { @@ -1679,7 +1674,6 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod if (!isLegacy) nodeEnv.FLUXOS_PATH = '/flux'; if (pausedSyncthingNodes.includes(i)) nodeEnv.FLUX_SYNCTHING_PAUSED = '1'; if (freezableDockerRoot.includes(i)) nodeEnv.FLUX_FREEZABLE_DOCKER_ROOT = 'true'; - if (freezableConfig.includes(i)) nodeEnv.FLUX_FREEZABLE_CONFIG = 'true'; if (syncthingFaults[i]) nodeEnv.FLUX_SYNCTHING_FAULT = syncthingFaults[i]; // WHETHER THIS NODE'S FLUXOS IS ROOT. Declared per node rather than derived from // legacy, because the field carries both: Arcane runs FluxOS as root, and an diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index a7848430e2..0afbe3afac 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -9,7 +9,7 @@ import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; import { buildSeedableTestApp } from '../framework/seed-helper.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { - freezeDocker, thawDocker, freezeConfig, thawConfig, FLUXOS_CONFIG, DOCKER_ROOT, + freezeDocker, thawDocker, DOCKER_ROOT, thawEveryNode, stuckThreads, processesThatCanRun, containerCgroup, stallDisk, resumeDisk, readsHeld, } from '../framework/host-stall.js'; @@ -32,9 +32,9 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // runs out. With docker frozen as well (systemd-control.freezeUnit), docker // cannot act at all, and the app is killed without docker. A node holding several // apps has every one of them stopped within that one window, long before even one -// removal's deadline. A node whose FluxOS config directory, where the trip is -// recorded, is frozen as well stops its apps all the same, and records the trip -// once the directory takes writes again. A node whose docker disk stops answering +// removal's deadline. A node whose FluxOS restarts while the stall is under way +// carries on counting it, and goes out of service when it would have without the +// restart. A node whose docker disk stops answering // reads as well, so docker's own records of its containers cannot be read, stops // and kills its apps through their slices within the same window. // @@ -54,10 +54,16 @@ const STOP_WINDOW_MS = 10 * 1000; const KILL_SLACK_MS = 10 * 1000; const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; const OWNER = 'hostStall'; -const ARCANE = [0, 1, 2]; +// the Arcane nodes the first case's app and the several apps' case share +const ARCANE = [0, 1]; +// an Arcane node whose FluxOS restarts mid-stall +const RESTARTED = 2; const LEGACY_UNPRIVILEGED = 3; // an Arcane node of its own: every case leaves its node out of service const DISK_STALLED = 4; +// The node whose FluxOS restarts mid-stall reads every RESTART_READ_INTERVAL_MS, so +// the readings a restart would lose take far longer than a restart does. +const RESTART_READ_INTERVAL_MS = 20 * 1000; const byOwner = (data) => data.owner === OWNER; const outOfServiceForTheStall = (data) => data.dosState >= 100 @@ -99,8 +105,8 @@ describe('a stalled host takes the node out of service', function () { hookCtx: this, nodes: 5, tickerAutostart: false, - freezableDockerRoot: [0, 1, 2, 3, DISK_STALLED], - freezableConfig: [0, 1, 2, 3], + freezableDockerRoot: [...ARCANE, RESTARTED, LEGACY_UNPRIVILEGED, DISK_STALLED], + nodeConfigOverrides: { [RESTARTED]: { hostStall: { readIntervalMs: RESTART_READ_INTERVAL_MS } } }, legacyNodes: [LEGACY_UNPRIVILEGED], unprivilegedNodes: [LEGACY_UNPRIVILEGED], configOverrides: { @@ -344,65 +350,53 @@ describe('a stalled host takes the node out of service', function () { }); }); - describe('a stall past the threshold, on a node whose config directory stalls too', function () { - const recordName = `e2ehoststallrec${Date.now()}`; - const record = `${FLUXOS_CONFIG}/hostStall.json`; - let recClient; - let recContainer; + describe('a stall past the threshold, on a node whose FluxOS restarts before the node goes out of service', function () { + const restartName = `e2ehoststallrestart${Date.now()}`; + let restartClient; + let restartContainer; let anchor; + let inServiceAtRestart; + let restartedAt; let trippedAt; - let stopped; - let recordedWhileFrozen; before(async function () { - this.timeout(480000); - // the Arcane node no other case here uses - const index = ARCANE.filter((i) => i !== node).at(-1); - recClient = env.clients[index]; - ({ container: recContainer } = recClient); - await installOn(env, recordName, [index]); - await waitForUp(recClient, recordName, `${recordName} is up`); - anchor = recClient.getLastEventId(); - step(`freezing node ${index}'s docker and its config directory`); - await freezeConfig(recContainer); - await freezeDocker(recContainer); - const trip = await recClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + this.timeout(600000); + restartClient = env.clients[RESTARTED]; + ({ container: restartContainer } = restartClient); + await installOn(env, restartName, [RESTARTED]); + await waitForUp(restartClient, restartName, `${restartName} is up`); + anchor = restartClient.getLastEventId(); + step(`freezing node ${RESTARTED}'s docker`); + await freezeDocker(restartContainer); + // the second reading finds docker held; three readings after it, the node is + // two readings short of going out of service + await restartClient.waitForEvent('placementHold:set', byOwner, RESTART_READ_INTERVAL_MS * 3, { afterId: anchor }); + await new Promise((resolve) => { setTimeout(resolve, RESTART_READ_INTERVAL_MS * 3.25); }); + inServiceAtRestart = (await restartClient.getDOSState()).data.dosState < 100; + const restartAnchor = restartClient.getLastEventId(); + restartedAt = Date.now(); + step('restarting FluxOS'); + await restartFluxos(restartContainer, { timeout: 120000, readyTimeoutMs: 180000 }); + const trip = await restartClient.waitForEvent('dos:changed', outOfServiceForTheStall, RESTART_READ_INTERVAL_MS * (STUCK_INTERVALS + 3), { afterId: restartAnchor }) .catch(() => null); trippedAt = trip ? Date.now() : null; - stopped = (await recClient.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS + KILL_SLACK_MS, { afterId: anchor }) - .catch(() => null))?.data ?? null; - recordedWhileFrozen = (await execInContainer(recContainer, `test -e ${record}`)).exitCode === 0; + step('out of service after the restart'); }); after(async function () { - if (recClient) await thawEveryNode([recClient]); - }); - - it('held the config directory: a write to it does not return', async function () { - // A write held on a frozen filesystem cannot be killed, so it is left to finish - // in the background, after the thaw. - const write = await execInContainer(recContainer, - `rm -f /tmp/probe-written; setsid sh -c 'touch ${FLUXOS_CONFIG}/.probe && touch /tmp/probe-written' /dev/null 2>&1 & sleep 3; test ! -e /tmp/probe-written`); - expect(write.exitCode, 'a write to the config directory returned, so the record was never held').to.equal(0); - }); - - it('takes the node out of service', function () { - expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + if (restartClient) await thawEveryNode([restartClient]); }); - it('stops its apps though the trip cannot be recorded', function () { - expect(recordedWhileFrozen, 'fixture: the trip was recorded on a frozen directory').to.equal(false); - expect(stopped, 'the node never stopped its apps while its trip could not be recorded').to.not.equal(null); - expect(stopped.apps.map((a) => a.app), 'the apps the node stopped').to.include(recordName); + it('held docker in the kernel across the restart, the node still in service when FluxOS restarted', async function () { + const stuck = await stuckThreads(restartContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the restart, so nothing below was tested').to.not.be.empty; + expect(inServiceAtRestart, 'fixture: the node was out of service before FluxOS restarted').to.equal(true); }); - it('records the trip once the config directory takes writes again', async function () { - this.timeout(60000); - await thawConfig(recContainer); - await thawDocker(recContainer); - await waitFor(async () => (await execInContainer(recContainer, `grep -q 'Host stall' ${record}`)).exitCode === 0, { - timeout: 30000, interval: 1000, label: 'the trip recorded once the config directory came back', - }); + it('goes out of service after the restart when it would have without it', function () { + expect(trippedAt, 'the node never went out of service after the restart').to.not.equal(null); + // two readings were left when FluxOS restarted; a count started again needs all of them + expect(trippedAt - restartedAt, 'the restart started the count of readings again').to.be.below(RESTART_READ_INTERVAL_MS * 3); }); }); From 22e379f79e33ac8daa0a2e476d08d8aa5fd60f2c Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 10:19:09 +0100 Subject: [PATCH 249/369] fix(apps): a host stall's state write overtaken by a stop leaves the kept state as it was A reading keeps its state in the file it started with, and a stop() that overtakes the write leaves the kept state as it was: the staged file is removed, never renamed into place. stop() resolves once the write in flight has ended, so a caller that removes the state's directory waits for it. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 28 +++++++++++++---- tests/unit/hostStallService.test.js | 40 ++++++++++++++++++++++-- 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 0b7266a944..44826c926c 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -45,6 +45,8 @@ let readTimer = null; // Advanced by stop(), so a reading still under way when the service stops // neither keeps what it read nor acts on it. let generation = 0; +/** The state being kept, which stop() lets its caller wait for. */ +let keeping = Promise.resolve(); /** Called once the node is out of service for a host stall. */ const tripListeners = []; const DEFAULT_PATHS = { @@ -166,14 +168,21 @@ function describe(thread) { /** * Keep the trip and every thread's count, so a FluxOS restart before the next - * reboot carries both on. + * reboot carries both on. A stop() that overtakes it leaves the kept state as it + * was. + * @param {number} startedIn The generation of the reading this keeps. + * @param {string} stateFile Where that reading keeps it. */ -async function keepState() { +async function keepState(startedIn, stateFile) { const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; + const staged = `${stateFile}.tmp`; try { - const staged = `${paths.stateFile}.tmp`; await fs.writeFile(staged, JSON.stringify({ trip, threads: [...tracked] })); - await fs.rename(staged, paths.stateFile); + if (startedIn !== generation) { + await fs.rm(staged, { force: true }); + return; + } + await fs.rename(staged, stateFile); } catch (error) { log.error(`hostStall - the state could not be kept: ${error.message}`); } @@ -234,11 +243,13 @@ async function act() { async function readOnce(options = {}) { usePaths(options); const startedIn = generation; + const { stateFile } = paths; const threads = await readDependencyThreads(paths.procRoot); if (startedIn !== generation) return; tracked = carryForward(threads); await act(); - await keepState(); + keeping = keepState(startedIn, stateFile); + await keeping; } async function readAndReschedule() { @@ -263,7 +274,11 @@ async function start(options = {}) { readTimer = setTimeout(readAndReschedule, 0); } -/** Take no more readings and forget every thread. */ +/** + * Take no more readings and forget every thread. + * @returns {Promise} Resolves once the state being kept, if any, is written + * or left. + */ function stop() { generation += 1; clearTimeout(readTimer); @@ -272,6 +287,7 @@ function stop() { tripReason = null; paths = { ...DEFAULT_PATHS }; tripListeners.length = 0; + return keeping; } module.exports = { diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 343d4022ee..f6bc6a4537 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -58,8 +58,8 @@ describe('hostStallService tests', () => { paths = { procRoot, stateFile: path.join(procRoot, 'flux-host-stall.json') }; }); - afterEach(() => { - hostStallService.stop(); + afterEach(async () => { + await hostStallService.stop(); fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); fs.rmSync(procRoot, { recursive: true, force: true }); @@ -238,7 +238,7 @@ describe('hostStallService tests', () => { describe('the kept state', () => { async function restart() { - hostStallService.stop(); + await hostStallService.stop(); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); await hostStallService.start(paths); } @@ -306,6 +306,40 @@ describe('hostStallService tests', () => { expect(outOfService()).to.equal(false); }); + describe('a stop while the state is being written', () => { + let release; + let reading; + beforeEach(async () => { + release = null; + const writeFile = fs.promises.writeFile.bind(fs.promises); + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.stateFile) + ? new Promise((resolve) => { release = () => resolve(writeFile(file, ...rest)); }) + : writeFile(file, ...rest))); + stuckMongod(); + reading = read(); + while (!release) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop + }); + + it('leaves the kept state as it was', async () => { + const stopped = hostStallService.stop(); + release(); + await Promise.all([stopped, reading]); + + expect(fs.readdirSync(procRoot).filter((entry) => entry.startsWith('flux-host-stall'))).to.deep.equal([]); + }); + + it('resolves once the write has ended', async () => { + let done = false; + const stopped = hostStallService.stop().then(() => { done = true; }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(done, 'the stop resolved before the write ended').to.equal(false); + + release(); + await stopped; + await reading; + }); + }); + it('takes the node out of service and tells its listeners when the state cannot be kept', async () => { const writeFile = fs.promises.writeFile.bind(fs.promises); sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.stateFile) From 0a309bf7a7d2c06212d1e8e76a335af014e4cbed Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 10:32:23 +0100 Subject: [PATCH 250/369] fix(apps): a host stall's state write in flight at a stop still lands stop() takes no more readings and resolves once the state being written has been kept, so a shutdown keeps the last reading and the trip with it. A write keeps its state in the file it started with. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 18 +++++------------- tests/unit/hostStallService.test.js | 4 ++-- 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 44826c926c..8be1595301 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -168,20 +168,14 @@ function describe(thread) { /** * Keep the trip and every thread's count, so a FluxOS restart before the next - * reboot carries both on. A stop() that overtakes it leaves the kept state as it - * was. - * @param {number} startedIn The generation of the reading this keeps. - * @param {string} stateFile Where that reading keeps it. + * reboot carries both on. */ -async function keepState(startedIn, stateFile) { +async function keepState() { const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; + const { stateFile } = paths; const staged = `${stateFile}.tmp`; try { await fs.writeFile(staged, JSON.stringify({ trip, threads: [...tracked] })); - if (startedIn !== generation) { - await fs.rm(staged, { force: true }); - return; - } await fs.rename(staged, stateFile); } catch (error) { log.error(`hostStall - the state could not be kept: ${error.message}`); @@ -243,12 +237,11 @@ async function act() { async function readOnce(options = {}) { usePaths(options); const startedIn = generation; - const { stateFile } = paths; const threads = await readDependencyThreads(paths.procRoot); if (startedIn !== generation) return; tracked = carryForward(threads); await act(); - keeping = keepState(startedIn, stateFile); + keeping = keepState(); await keeping; } @@ -276,8 +269,7 @@ async function start(options = {}) { /** * Take no more readings and forget every thread. - * @returns {Promise} Resolves once the state being kept, if any, is written - * or left. + * @returns {Promise} Resolves once the state being kept, if any, is written. */ function stop() { generation += 1; diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index f6bc6a4537..59b0781ba0 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -320,12 +320,12 @@ describe('hostStallService tests', () => { while (!release) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop }); - it('leaves the kept state as it was', async () => { + it('still keeps the state being written', async () => { const stopped = hostStallService.stop(); release(); await Promise.all([stopped, reading]); - expect(fs.readdirSync(procRoot).filter((entry) => entry.startsWith('flux-host-stall'))).to.deep.equal([]); + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); }); it('resolves once the write has ended', async () => { From c71dccd066f837c992d09087261128408beb00b0 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 12:53:08 +0100 Subject: [PATCH 251/369] test(harness): a dockerd restart leaves a node's FluxOS running Co-Authored-By: Claude Opus 5.5 --- test-infra/systemd/flux-node-network.service | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test-infra/systemd/flux-node-network.service b/test-infra/systemd/flux-node-network.service index 44ee2ebed3..b652973207 100644 --- a/test-infra/systemd/flux-node-network.service +++ b/test-infra/systemd/flux-node-network.service @@ -1,9 +1,10 @@ [Unit] Description=Node network shape and global IPv6 (harness node) # Built once dockerd answers and before FluxOS starts: a node reads its network at -# boot, and dockerd refuses to start under the routes a shape adds. +# boot, and dockerd refuses to start under the routes a shape adds. Wanted, not +# required: a dockerd restart leaves this unit and FluxOS running, as on a node. After=dockerd.service -Requires=dockerd.service +Wants=dockerd.service Before=fluxos.service pm2-fluxuser.service [Service] From 5d4876d8e48fe51e5a383c9ac08ee8b61d3800a2 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 12:53:08 +0100 Subject: [PATCH 252/369] test(harness): a legacy node's syncthing runs on through a FluxOS stop or death, as under pm2 Co-Authored-By: Claude Opus 5.5 --- test-infra/entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 96b8a9c511..161da4c7c5 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -315,6 +315,13 @@ if [ -n "$FLUX_FLUXOS_USER" ]; then printf '[Service]\nUser=%s\n' "$FLUX_FLUXOS_USER" > /etc/systemd/system/fluxos.service.d/user.conf chown "$(id -u "$FLUX_FLUXOS_USER"):$(id -g "$FLUX_FLUXOS_USER")" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" fi +# A legacy node's FluxOS starts syncthing outside its own process tree, and pm2, +# which runs FluxOS on a legacy node, signals only that tree when FluxOS stops or +# dies: syncthing runs on. The fluxos unit signals FluxOS's own process alone. +if [ -z "$FLUXOS_PATH" ] && [ "$FLUX_PM2" != "1" ]; then + mkdir -p /etc/systemd/system/fluxos.service.d + printf '[Service]\nKillMode=process\n' > /etc/systemd/system/fluxos.service.d/legacy.conf +fi if [ -n "$FLUX_SYNCTHING_HOST" ]; then ln -sf /etc/systemd/system/syncthing-forward.service /etc/systemd/system/multi-user.target.wants/syncthing-forward.service fi From e6c137713785e1f4a9091890d7e51dc53c256c88 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 12:53:08 +0100 Subject: [PATCH 253/369] test(harness): 02 reads the node's dockerd from its unit Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/tests/02-boot-prerequisites.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/02-boot-prerequisites.js b/test-infra/runner/tests/02-boot-prerequisites.js index 5c67e993d6..182196b554 100644 --- a/test-infra/runner/tests/02-boot-prerequisites.js +++ b/test-infra/runner/tests/02-boot-prerequisites.js @@ -3,6 +3,7 @@ import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { waitForDaemonReady, waitFor } from '../framework/wait.js'; import { dbClient } from '../framework/db-client.js'; +import { execInContainer } from '../framework/container.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; @@ -67,7 +68,9 @@ describe('Boot: prerequisites', function () { }); it('should start Docker daemon inside container', async function () { - expect(env.nodeHasLog(0, 'dockerd is ready')).to.equal(true); + const unit = await execInContainer(env.clients[0].container, 'systemctl is-active dockerd'); + expect(unit.stdout.trim()).to.equal('active'); + expect(env.nodeHasLog(0, 'Docker daemon connected')).to.equal(true); }); it('should initialize app spawner', async function () { From 994ef5fddcad7e1dc70c204ece0918f1b461041f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 13:48:20 +0100 Subject: [PATCH 254/369] fix(apps): a host stall's count moves only once an interval has passed by the machine's uptime, so FluxOS restarts add nothing Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 47 +++++++++++--- tests/unit/hostStallService.test.js | 79 ++++++++++++++++++++++-- 2 files changed, 114 insertions(+), 12 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index 8be1595301..d8b8f4ceec 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -9,7 +9,10 @@ * the node goes out of service until it reboots. * * Readings are counted rather than time measured: a VM its hypervisor paused - * takes no readings while it is paused, so a pause cannot read as a stall. + * takes no readings while it is paused, so a pause cannot read as a stall. A + * reading counts only when one interval has passed, by the machine's uptime, + * since the reading the thread's count last moved at: FluxOS takes a reading as + * it starts, and a start soon after another adds nothing. * * Out of service lasts until the machine reboots, through any number of FluxOS * restarts, and so does the count of readings toward it. Both are kept in a file @@ -32,11 +35,14 @@ const fluxNetworkHelper = require('./fluxNetworkHelper'); const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); const READ_INTERVAL_MS = config.hostStall.readIntervalMs; const STUCK_INTERVALS = config.hostStall.stuckIntervals; +// /proc/uptime is written in hundredths of a second. +const UPTIME_RESOLUTION_MS = 10; /** * Every dependency thread in D at the last reading, keyed by process, thread - * and the thread's start time so a reused id is a new thread. - * @type {Map} + * and the thread's start time so a reused id is a new thread, with its count + * and the uptime its count last moved at. + * @type {Map} */ let tracked = new Map(); /** Why the node is out of service for a host stall, once it is. */ @@ -138,20 +144,40 @@ async function readDependencyThreads(procRoot) { return threads; } +/** + * The machine's uptime, which runs on through FluxOS restarts and starts again + * only at a boot. + * @param {string} procRoot + * @returns {Promise} Milliseconds, or null when it cannot be read. + */ +async function readUptimeMs(procRoot) { + const seconds = Number((await readOrNull(path.join(procRoot, 'uptime')))?.split(' ')[0]); + return Number.isFinite(seconds) ? seconds * 1000 : null; +} + /** * Carry each thread's count of intervals stuck from the last reading to this - * one. A thread that ran, or left D, starts again. + * one. A thread that did not run adds one once an interval has passed since its + * count last moved, and keeps its count before then. A thread that ran, or left + * D, starts again. * @param {Array} threads One reading. + * @param {number} nowMs The uptime the reading was taken at. * @returns {Map} */ -function carryForward(threads) { +function carryForward(threads, nowMs) { const next = new Map(); threads.filter((thread) => thread.state === 'D').forEach((thread) => { const key = `${thread.process}/${thread.pid}/${thread.tid}/${thread.startTime}`; const before = tracked.get(key); - const intervals = before && before.switches === thread.switches ? before.intervals + 1 : 0; + let intervals = 0; + let countedAtMs = nowMs; + if (before && before.switches === thread.switches) { + const intervalPassed = nowMs - before.countedAtMs + UPTIME_RESOLUTION_MS >= READ_INTERVAL_MS; + intervals = intervalPassed ? before.intervals + 1 : before.intervals; + countedAtMs = intervalPassed ? nowMs : before.countedAtMs; + } next.set(key, { - process: thread.process, tid: thread.tid, wchan: thread.wchan, switches: thread.switches, intervals, + process: thread.process, tid: thread.tid, wchan: thread.wchan, switches: thread.switches, intervals, countedAtMs, }); }); return next; @@ -237,9 +263,14 @@ async function act() { async function readOnce(options = {}) { usePaths(options); const startedIn = generation; + const nowMs = await readUptimeMs(paths.procRoot); + if (nowMs === null) { + log.error('hostStall - the uptime could not be read; this reading is not taken'); + return; + } const threads = await readDependencyThreads(paths.procRoot); if (startedIn !== generation) return; - tracked = carryForward(threads); + tracked = carryForward(threads, nowMs); await act(); keeping = keepState(); await keeping; diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 59b0781ba0..c7fc7a112c 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -7,7 +7,7 @@ const config = require('config'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const hostStallService = require('../../ZelBack/src/services/hostStallService'); -const { stuckIntervals } = config.hostStall; +const { stuckIntervals, readIntervalMs } = config.hostStall; describe('hostStallService tests', () => { let procRoot; @@ -41,11 +41,30 @@ describe('hostStallService tests', () => { } const stuckMongod = (switches = 40) => processWith('100', 'mongod', [{ tid: '101', state: 'D', switches }, { tid: '102', state: 'S', switches: 5 }]); - const read = () => hostStallService.readOnce(paths); + // The machine's uptime in the fake /proc, which a reading reads. + let uptimeMs; + function setUptime(ms) { + uptimeMs = ms; + fs.writeFileSync(path.join(procRoot, 'uptime'), `${(ms / 1000).toFixed(2)} 0.00\n`); + } + const passMs = (ms) => setUptime(uptimeMs + ms); + // A reading one interval after the last, as the service's own timer takes them. + const read = () => { + passMs(readIntervalMs); + return hostStallService.readOnce(paths); + }; const kept = () => JSON.parse(fs.readFileSync(paths.stateFile, 'utf8')); async function readTimes(n) { for (let i = 0; i < n; i += 1) await read(); // eslint-disable-line no-await-in-loop } + // A start takes a reading of its own, as soon as it has restored the kept state; + // this waits for that reading to be kept, so no other reading runs beside it. + const keptAtMs = () => (fs.existsSync(paths.stateFile) ? fs.statSync(paths.stateFile).mtimeMs : null); + async function startAndRead() { + const before = keptAtMs(); + await hostStallService.start(paths); + while (keptAtMs() === before) await new Promise((resolve) => { setTimeout(resolve, 1); }); // eslint-disable-line no-await-in-loop + } const placementHeld = () => fluxNetworkHelper.isPlacementHeld(); const outOfService = () => fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL); @@ -56,6 +75,7 @@ describe('hostStallService tests', () => { fs.symlinkSync('pid:[4026531836]', path.join(procRoot, 'self', 'ns', 'pid')); fs.writeFileSync(path.join(procRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); paths = { procRoot, stateFile: path.join(procRoot, 'flux-host-stall.json') }; + setUptime(1000000); }); afterEach(async () => { @@ -72,6 +92,7 @@ describe('hostStallService tests', () => { const otherRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-other-')); fs.mkdirSync(path.join(otherRoot, 'self')); fs.writeFileSync(path.join(otherRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); + fs.writeFileSync(path.join(otherRoot, 'uptime'), '1000.00 0.00\n'); let finishReading = null; const readdir = fs.promises.readdir.bind(fs.promises); sinon.stub(fs.promises, 'readdir').callsFake((dir, ...rest) => (dir === otherRoot @@ -230,7 +251,7 @@ describe('hostStallService tests', () => { const listener = sinon.stub(); hostStallService.onTrip(listener); - await hostStallService.start(paths); + await startAndRead(); sinon.assert.calledOnce(listener); }); @@ -240,7 +261,7 @@ describe('hostStallService tests', () => { async function restart() { await hostStallService.stop(); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); - await hostStallService.start(paths); + await startAndRead(); } it('keeps the trip and every thread\'s count after each reading', async () => { @@ -356,6 +377,56 @@ describe('hostStallService tests', () => { }); }); + describe('the time between readings', () => { + it('adds nothing for a reading taken before an interval has passed since the count moved', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + passMs(readIntervalMs - 1000); + + await hostStallService.readOnce(paths); + + expect(outOfService()).to.equal(false); + expect(kept().threads.map(([, thread]) => thread.intervals)).to.deep.equal([stuckIntervals - 1]); + }); + + it('counts a reading one interval after the count moved, by the uptime\'s resolution', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + passMs(readIntervalMs - 10); + + await hostStallService.readOnce(paths); + + expect(outOfService()).to.equal(true); + }); + + it('goes out of service only once the thread has been held for the configured intervals, however often FluxOS restarts', async () => { + const RESTART_EVERY_MS = 15000; + stuckMongod(); + await read(); + const heldSince = uptimeMs; + while (!outOfService()) { + await hostStallService.stop(); // eslint-disable-line no-await-in-loop + passMs(RESTART_EVERY_MS); + await startAndRead(); // eslint-disable-line no-await-in-loop + expect(uptimeMs - heldSince, 'never went out of service').to.be.below(2 * stuckIntervals * readIntervalMs); + } + + expect(uptimeMs - heldSince).to.be.within(stuckIntervals * readIntervalMs, stuckIntervals * readIntervalMs + RESTART_EVERY_MS); + }); + + it('takes no reading while the uptime cannot be read', async () => { + stuckMongod(); + await readTimes(2); + fs.rmSync(path.join(procRoot, 'uptime')); + stuckMongod(41); + + await hostStallService.readOnce(paths); + + expect(kept().threads.map(([, thread]) => thread.intervals)).to.deep.equal([1]); + expect(placementHeld()).to.equal(true); + }); + }); + it('names no wait point the kernel does not show', async () => { processWith('700', 'dockerd', [{ tid: '701', state: 'D', switches: 9, wchan: '0' }]); From 548d0f191e477fe2d14e5163ebf5b660a18df9d1 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 13:49:17 +0100 Subject: [PATCH 255/369] test(harness): 3303 restarts FluxOS over and over mid-stall and the node stays in service Co-Authored-By: Claude Opus 5.5 --- ...lled-host-takes-the-node-out-of-service.js | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js index 0afbe3afac..e28616a053 100644 --- a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -34,7 +34,8 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // apps has every one of them stopped within that one window, long before even one // removal's deadline. A node whose FluxOS restarts while the stall is under way // carries on counting it, and goes out of service when it would have without the -// restart. A node whose docker disk stops answering +// restart; one whose FluxOS restarts over and over adds nothing to the count for +// the restarts. A node whose docker disk stops answering // reads as well, so docker's own records of its containers cannot be read, stops // and kills its apps through their slices within the same window. // @@ -64,6 +65,13 @@ const DISK_STALLED = 4; // The node whose FluxOS restarts mid-stall reads every RESTART_READ_INTERVAL_MS, so // the readings a restart would lose take far longer than a restart does. const RESTART_READ_INTERVAL_MS = 20 * 1000; +// A node whose FluxOS restarts over and over while its docker is held, every +// restart well inside one reading interval, so restarts that each counted as one +// would take it out of service long before the readings could. +const CRASH_LOOPED = 5; +const CRASH_LOOP_READ_INTERVAL_MS = 180 * 1000; +// a start's reading for each restart would reach the threshold with one to spare +const CRASH_LOOP_RESTARTS = STUCK_INTERVALS + 2; const byOwner = (data) => data.owner === OWNER; const outOfServiceForTheStall = (data) => data.dosState >= 100 @@ -103,10 +111,13 @@ describe('a stalled host takes the node out of service', function () { this.timeout(480000); env = await createTestEnv({ hookCtx: this, - nodes: 5, + nodes: 6, tickerAutostart: false, - freezableDockerRoot: [...ARCANE, RESTARTED, LEGACY_UNPRIVILEGED, DISK_STALLED], - nodeConfigOverrides: { [RESTARTED]: { hostStall: { readIntervalMs: RESTART_READ_INTERVAL_MS } } }, + freezableDockerRoot: [...ARCANE, RESTARTED, LEGACY_UNPRIVILEGED, DISK_STALLED, CRASH_LOOPED], + nodeConfigOverrides: { + [RESTARTED]: { hostStall: { readIntervalMs: RESTART_READ_INTERVAL_MS } }, + [CRASH_LOOPED]: { hostStall: { readIntervalMs: CRASH_LOOP_READ_INTERVAL_MS } }, + }, legacyNodes: [LEGACY_UNPRIVILEGED], unprivilegedNodes: [LEGACY_UNPRIVILEGED], configOverrides: { @@ -400,6 +411,46 @@ describe('a stalled host takes the node out of service', function () { }); }); + describe('a stall on a node whose FluxOS restarts over and over', function () { + const loopName = `e2ehoststallloop${Date.now()}`; + let loopClient; + let loopContainer; + let restartsTookMs; + + before(async function () { + this.timeout(900000); + loopClient = env.clients[CRASH_LOOPED]; + ({ container: loopContainer } = loopClient); + await installOn(env, loopName, [CRASH_LOOPED]); + await waitForUp(loopClient, loopName, `${loopName} is up`); + step(`freezing node ${CRASH_LOOPED}'s docker`); + await freezeDocker(loopContainer); + const from = Date.now(); + for (let i = 0; i < CRASH_LOOP_RESTARTS; i += 1) { + step(`restarting FluxOS, ${i + 1} of ${CRASH_LOOP_RESTARTS}`); + // eslint-disable-next-line no-await-in-loop + await restartFluxos(loopContainer, { timeout: 120000, readyTimeoutMs: 180000 }); + } + restartsTookMs = Date.now() - from; + }); + + after(async function () { + if (loopClient) await thawEveryNode([loopClient]); + }); + + it('held docker in the kernel across the restarts, every one of them inside two readings', async function () { + const stuck = await stuckThreads(loopContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the restarts, so nothing below was tested').to.not.be.empty; + expect(restartsTookMs, 'fixture: the restarts took long enough for the readings alone to count').to.be.below(CRASH_LOOP_READ_INTERVAL_MS * 2); + }); + + // Out of service for a stall lasts through every restart, so the state after the + // last one says whether any of them took the node out. + it('stays in service, the restarts adding nothing to the count', async function () { + expect((await loopClient.getDOSState()).data.dosState, 'the restarts took the node out of service').to.be.below(100); + }); + }); + describe('a stall past the threshold, on a node whose docker disk stops answering reads', function () { const diskName = `e2ehoststalldisk${Date.now()}`; let diskClient; From 6c4470b322e12cfa8eb783e5c8896ab1b8510968 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:03:39 +0100 Subject: [PATCH 256/369] fix(apps): a host stall's state is kept in /run/fluxos, and written only when it changes Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 59 +++++++++-- tests/unit/hostStallService.test.js | 125 +++++++++++++++++++++-- 2 files changed, 162 insertions(+), 22 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index d8b8f4ceec..f40d306696 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -15,10 +15,11 @@ * it starts, and a start soon after another adds nothing. * * Out of service lasts until the machine reboots, through any number of FluxOS - * restarts, and so does the count of readings toward it. Both are kept in a file - * on /dev/shm, the tmpfs every Linux mounts at boot: memory, so reading it cannot - * stall with a disk, and emptied by a reboot and by nothing else. Not a database - * row, because the database may be what is stuck. + * restarts, and so does the count of readings toward it. Both are kept in + * /run/fluxos, on the tmpfs every Linux mounts at /run at boot: memory, so + * reading it cannot stall with a disk, and emptied by a reboot. Only root and + * FluxOS's own account may write there. Not a database row, because the database + * may be what is stuck. The file is written only when what it holds changes. * * WHAT IS READ. The threads of the processes FluxOS depends on, in FluxOS's own * PID namespace. An app container that runs one of the same programs has its @@ -27,9 +28,11 @@ */ const fs = require('fs').promises; +const os = require('os'); const path = require('path'); const config = require('config'); const log = require('../lib/log'); +const serviceHelper = require('./serviceHelper'); const fluxNetworkHelper = require('./fluxNetworkHelper'); const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); @@ -47,6 +50,7 @@ const UPTIME_RESOLUTION_MS = 10; let tracked = new Map(); /** Why the node is out of service for a host stall, once it is. */ let tripReason = null; +let running = false; let readTimer = null; // Advanced by stop(), so a reading still under way when the service stops // neither keeps what it read nor acts on it. @@ -57,9 +61,13 @@ let keeping = Promise.resolve(); const tripListeners = []; const DEFAULT_PATHS = { procRoot: '/proc', - stateFile: '/dev/shm/flux-host-stall.json', + stateFile: '/run/fluxos/host-stall.json', }; let paths = { ...DEFAULT_PATHS }; +// What the state file holds with nothing stuck and no trip. +const NOTHING_KEPT = JSON.stringify({ trip: null, threads: [] }); +/** What the state file holds, as this process last wrote or found it; null when unknown. */ +let keptText = NOTHING_KEPT; /** * @param {{procRoot?: string, stateFile?: string}} options @@ -192,17 +200,41 @@ function describe(thread) { return `${thread.process} thread ${thread.tid} has been held in the kernel at ${thread.wchan} for ${minutes} minutes`; } +/** + * Make the state file's directory. /run is root's: a FluxOS that is not root + * makes it through sudo, owned by its own account, and writes in it directly. + * @param {string} dir + */ +async function makeStateDirectory(dir) { + try { + await fs.mkdir(dir, { recursive: true, mode: 0o755 }); + return; + } catch (error) { + if (error.code !== 'EACCES' && error.code !== 'EPERM') throw error; + } + const { uid, gid } = os.userInfo(); + const { error } = await serviceHelper.runCommand('install', { + runAsRoot: true, + params: ['-d', '-m', '755', '-o', String(uid), '-g', String(gid), dir], + }); + if (error) throw error; +} + /** * Keep the trip and every thread's count, so a FluxOS restart before the next - * reboot carries both on. + * reboot carries both on. Nothing is written when the file already holds them. */ async function keepState() { const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; + const text = JSON.stringify({ trip, threads: [...tracked] }); + if (text === keptText) return; const { stateFile } = paths; const staged = `${stateFile}.tmp`; try { - await fs.writeFile(staged, JSON.stringify({ trip, threads: [...tracked] })); + await makeStateDirectory(path.dirname(stateFile)); + await fs.writeFile(staged, text); await fs.rename(staged, stateFile); + keptText = text; } catch (error) { log.error(`hostStall - the state could not be kept: ${error.message}`); } @@ -227,10 +259,12 @@ function onTrip(listener) { /** Restore what an earlier FluxOS in this boot kept: the trip, and every thread's count. */ async function restoreState() { const text = await readOrNull(paths.stateFile); + keptText = text === null ? NOTHING_KEPT : null; if (text === null) return; try { const { trip, threads } = JSON.parse(text); tracked = new Map(threads); + keptText = text; if (trip) { tripReason = trip; fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, trip); @@ -282,20 +316,21 @@ async function readAndReschedule() { } catch (error) { log.error(`hostStall - reading the dependency threads failed: ${error.message}`); } - if (readTimer !== null) readTimer = setTimeout(readAndReschedule, READ_INTERVAL_MS); + if (running) readTimer = setTimeout(readAndReschedule, READ_INTERVAL_MS); } /** * Restore what an earlier FluxOS in this boot kept, then read now, and every * READ_INTERVAL_MS after each reading ends. * @param {object} [options] Paths, as usePaths takes them. - * @returns {Promise} Resolves once the kept state is restored. + * @returns {Promise} Resolves once the kept state is restored and the first reading taken. */ async function start(options = {}) { - if (readTimer !== null) return; + if (running) return; + running = true; usePaths(options); await restoreState(); - readTimer = setTimeout(readAndReschedule, 0); + await readAndReschedule(); } /** @@ -304,10 +339,12 @@ async function start(options = {}) { */ function stop() { generation += 1; + running = false; clearTimeout(readTimer); readTimer = null; tracked = new Map(); tripReason = null; + keptText = NOTHING_KEPT; paths = { ...DEFAULT_PATHS }; tripListeners.length = 0; return keeping; diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index c7fc7a112c..e977a9ef6d 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -4,6 +4,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const config = require('config'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const hostStallService = require('../../ZelBack/src/services/hostStallService'); @@ -57,14 +58,6 @@ describe('hostStallService tests', () => { async function readTimes(n) { for (let i = 0; i < n; i += 1) await read(); // eslint-disable-line no-await-in-loop } - // A start takes a reading of its own, as soon as it has restored the kept state; - // this waits for that reading to be kept, so no other reading runs beside it. - const keptAtMs = () => (fs.existsSync(paths.stateFile) ? fs.statSync(paths.stateFile).mtimeMs : null); - async function startAndRead() { - const before = keptAtMs(); - await hostStallService.start(paths); - while (keptAtMs() === before) await new Promise((resolve) => { setTimeout(resolve, 1); }); // eslint-disable-line no-await-in-loop - } const placementHeld = () => fluxNetworkHelper.isPlacementHeld(); const outOfService = () => fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL); @@ -251,7 +244,7 @@ describe('hostStallService tests', () => { const listener = sinon.stub(); hostStallService.onTrip(listener); - await startAndRead(); + await hostStallService.start(paths); sinon.assert.calledOnce(listener); }); @@ -261,7 +254,7 @@ describe('hostStallService tests', () => { async function restart() { await hostStallService.stop(); fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); - await startAndRead(); + await hostStallService.start(paths); } it('keeps the trip and every thread\'s count after each reading', async () => { @@ -361,6 +354,116 @@ describe('hostStallService tests', () => { }); }); + describe('what is written', () => { + const writesOfState = () => fs.promises.writeFile.getCalls().filter((call) => String(call.args[0]).startsWith(paths.stateFile)).length; + beforeEach(() => { + sinon.spy(fs.promises, 'writeFile'); + }); + + it('writes nothing while nothing is stuck', async () => { + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 40 }]); + + await readTimes(3); + + expect(fs.existsSync(paths.stateFile)).to.equal(false); + }); + + it('writes nothing at a start with nothing kept while nothing is stuck', async () => { + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 40 }]); + + await restart(); + await readTimes(2); + + expect(fs.existsSync(paths.stateFile)).to.equal(false); + }); + + it('writes again at the next reading when a write fails', async () => { + const rename = fs.promises.rename.bind(fs.promises); + const renames = sinon.stub(fs.promises, 'rename').callsFake((from, ...rest) => rename(from, ...rest)); + renames.onFirstCall().rejects(new Error('no space left on device')); + stuckMongod(); + await read(); + + await hostStallService.readOnce(paths); + + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + + it('writes nothing for a reading that changes nothing', async () => { + stuckMongod(); + await read(); + const writes = writesOfState(); + + await hostStallService.readOnce(paths); + + expect(writesOfState()).to.equal(writes); + }); + + it('writes nothing at a start that finds what it kept unchanged, and writes once something changes', async () => { + stuckMongod(); + await readTimes(2); + const writes = writesOfState(); + await restart(); + expect(writesOfState(), 'the start wrote back what it had just read').to.equal(writes); + + await read(); + + expect(writesOfState()).to.equal(writes + 1); + }); + + it('writes again once the stuck thread runs', async () => { + stuckMongod(); + await readTimes(2); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 41 }]); + + await read(); + + expect(kept().threads).to.deep.equal([]); + }); + }); + + describe('a FluxOS that may not make the state directory', () => { + let stateDir; + beforeEach(() => { + stateDir = path.join(procRoot, 'run-fluxos'); + paths.stateFile = path.join(stateDir, 'host-stall.json'); + const mkdir = fs.promises.mkdir.bind(fs.promises); + sinon.stub(fs.promises, 'mkdir').callsFake((dir, ...rest) => (dir === stateDir && !fs.existsSync(stateDir) + ? Promise.reject(Object.assign(new Error('permission denied'), { code: 'EACCES' })) + : mkdir(dir, ...rest))); + }); + + const makesTheDirectory = async () => { + fs.mkdirSync(stateDir); + return { error: null, stdout: '', stderr: '' }; + }; + + it('makes it through sudo, owned by its own account, and keeps the state in it', async () => { + sinon.stub(serviceHelper, 'runCommand').callsFake(makesTheDirectory); + const { uid, gid } = os.userInfo(); + stuckMongod(); + + await read(); + + sinon.assert.calledOnceWithExactly(serviceHelper.runCommand, 'install', { + runAsRoot: true, params: ['-d', '-m', '755', '-o', String(uid), '-g', String(gid), stateDir], + }); + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + + it('tries again at the next reading when sudo fails', async () => { + const runCommand = sinon.stub(serviceHelper, 'runCommand').resolves({ error: new Error('sudo: a password is required'), stdout: '', stderr: '' }); + stuckMongod(); + await read(); + expect(fs.existsSync(paths.stateFile), 'fixture: the state was kept though sudo failed').to.equal(false); + runCommand.callsFake(makesTheDirectory); + + await hostStallService.readOnce(paths); + + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + }); + it('takes the node out of service and tells its listeners when the state cannot be kept', async () => { const writeFile = fs.promises.writeFile.bind(fs.promises); sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.stateFile) @@ -407,7 +510,7 @@ describe('hostStallService tests', () => { while (!outOfService()) { await hostStallService.stop(); // eslint-disable-line no-await-in-loop passMs(RESTART_EVERY_MS); - await startAndRead(); // eslint-disable-line no-await-in-loop + await hostStallService.start(paths); // eslint-disable-line no-await-in-loop expect(uptimeMs - heldSince, 'never went out of service').to.be.below(2 * stuckIntervals * readIntervalMs); } From 8ed1ffd3dde3d67cb0b72f0dfab6007b46db8856 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:09:36 +0100 Subject: [PATCH 257/369] fix(apps): a host stall's unload reads the installed apps and docker's records at once Co-Authored-By: Claude Opus 5.5 --- .../services/appMonitoring/nodeStatusMonitor.js | 7 +++++-- tests/unit/nodeStatusMonitor.test.js | 14 ++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index d7eb038938..621fc0f99e 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -143,9 +143,12 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { if (unloadingForHostStall) return; unloadingForHostStall = true; try { - const listed = await withinDeadline(installedAppsFn(), REMOVAL_DEADLINE_MS).catch(() => null); + // Read at once: a stall that holds one of them is likely to hold the other. + const [listed, read] = await Promise.all([ + withinDeadline(installedAppsFn(), REMOVAL_DEADLINE_MS).catch(() => null), + withinDeadline(appKill.readContainers(), STOP_WINDOW_MS), + ]); const installed = listed?.status === 'success' ? listed.data.map((app) => app.name) : []; - const read = await withinDeadline(appKill.readContainers(), STOP_WINDOW_MS); const containers = Array.isArray(read) ? read : null; if (!containers) log.error(`monitorNodeStatus - host stall: docker's container records could not be read (${read === TIMED_OUT ? 'no answer' : 'refused'}); containers outside the app slices cannot be found`); const names = [...new Set([...installed, ...await appKill.appsOnDisk(containers)])]; diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 704c54b9e6..7eb6cb0be6 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -370,6 +370,20 @@ describe('nodeStatusMonitor tests', () => { await unload; }); + it('reads the installed apps and docker\'s records at once, so a stall that holds both delays the stop by one wait', async () => { + const after = (ms, value) => () => new Promise((resolve) => { setTimeout(resolve, ms, value); }); + const installedAppsFn = sinon.stub().callsFake(after(STOP_WINDOW - 1, { status: 'success', data: [{ name: 'a' }] })); + appKill.readContainers.callsFake(after(STOP_WINDOW - 1, [])); + dockerService.appDockerStop.callsFake(never); + + const unload = monitor.unloadForHostStall(installedAppsFn, sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW - 1); + + sinon.assert.calledWith(dockerService.appDockerStop, 'fluxa'); + await clock.tickAsync(STOP_WINDOW); + await unload; + }); + it('names a refusal to list docker\'s records', async () => { appKill.readContainers.resolves(null); From c23543d4af4e03337de2c6684db1ef2898b5a751 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:10:20 +0100 Subject: [PATCH 258/369] fix(apps): a host stall's unload log names the one stop window Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appMonitoring/nodeStatusMonitor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 621fc0f99e..620193ebf9 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -152,7 +152,7 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { const containers = Array.isArray(read) ? read : null; if (!containers) log.error(`monitorNodeStatus - host stall: docker's container records could not be read (${read === TIMED_OUT ? 'no answer' : 'refused'}); containers outside the app slices cannot be found`); const names = [...new Set([...installed, ...await appKill.appsOnDisk(containers)])]; - log.warn(`monitorNodeStatus - host stall: stopping ${names.length} applications, ${STOP_WINDOW_MS / 1000}s between them`); + log.warn(`monitorNodeStatus - host stall: stopping ${names.length} applications within one ${STOP_WINDOW_MS / 1000}s window`); await withinDeadline(Promise.allSettled(names.map((name) => stopApp(name, containers))), STOP_WINDOW_MS); const stopped = new Map(); From 27fc43a9e19a4d588135f9de0837b8d227b88335 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:15:02 +0100 Subject: [PATCH 259/369] fix(apps): a host stall kills every container outside the app slices when docker's records cannot be read Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 59 +++++++++++--------- tests/unit/appKill.test.js | 27 +++++++-- 2 files changed, 56 insertions(+), 30 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index 15a45733af..7d227c432a 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -8,10 +8,10 @@ * read from the runtime's own records on disk (readContainers), which stay * readable while docker does not answer but not while its disk does not. The * records are read once and passed to every call here; null stands for records - * that could not be read, and then a container outside the slices cannot be - * named, killed or ruled out. Where the records are is read from docker at boot; - * until then it is docker's default for the node's kind. Where each cgroup is, - * systemd says (appSlice.controlGroup). + * that could not be read, and then every container outside the slices counts as + * each app's: every container on a node is an app's. Where the records are is + * read from docker at boot; until then it is docker's default for the node's + * kind. Where each cgroup is, systemd says (appSlice.controlGroup). * * A process held in the kernel is not removed by a kill: it keeps the kill * pending and dies the moment the kernel lets it go, without running the app's @@ -134,16 +134,36 @@ function containersOf(appName, containers) { return recordsOf(appName, containers).map(({ name }) => name); } +/** + * The scope of every container outside the app slices: directly under + * system.slice, where the runtime puts a container with no slice of its own. + * + * Such a container was created before FluxOS gave every app a slice + * (dockerService sets each container's CgroupParent); FluxOS creates none + * outside one. Once no node holds one, this, and the part docker's records play + * in finding an app's processes, can go. + * @returns {Promise} Their unit names; null when systemd does not answer. + */ +async function scopesOutsideSlices() { + const parent = await appSlice.controlGroup('system.slice'); + if (!parent) return null; + const entries = await fs.readdir(parent).catch(() => null); + return entries === null ? null : entries.filter((entry) => /^docker-[0-9a-f]{64}\.scope$/.test(entry)); +} + /** * Every cgroup that holds a container of the app and exists now: its slice, and - * the scope of each of its containers the records name. + * the scope of each of its containers the records name; with no records, the + * scope of every container outside the slices. * @param {string} appName * @param {ContainerRecord[]|null} containers readContainers - * @returns {Promise} + * @returns {Promise} Null when the records could not be read and + * systemd does not say which containers are outside the slices. */ async function cgroupsOf(appName, containers) { const unit = appSlice.sliceName(appName); - const scopes = recordsOf(appName, containers).map(({ id }) => `docker-${id}.scope`); + const scopes = containers ? recordsOf(appName, containers).map(({ id }) => `docker-${id}.scope`) : await scopesOutsideSlices(); + if (scopes === null) return null; const units = [...(unit ? [unit] : []), ...scopes]; const found = (await Promise.all(units.map(appSlice.controlGroup))).filter(Boolean); // a scope inside the app's slice is read and killed with the slice @@ -159,7 +179,8 @@ async function cgroupsOf(appName, containers) { * @returns {Promise} The cgroups it was sent to. */ async function kill(appName, containers) { - const cgroups = await cgroupsOf(appName, containers); + const cgroups = await cgroupsOf(appName, containers) + ?? await cgroupsOf(appName, []); // eslint-disable-next-line no-restricted-syntax for (const dir of cgroups) { // eslint-disable-next-line no-await-in-loop @@ -196,34 +217,22 @@ async function processesUnder(dir) { return [...own, ...below.flat()]; } -/** - * Whether any container lives outside the app slices: in a scope directly under - * system.slice, where the runtime puts a container with no slice of its own. - * @returns {Promise} True as well when systemd does not answer. - */ -async function anyContainerOutsideSlices() { - const parent = await appSlice.controlGroup('system.slice'); - if (!parent) return true; - const entries = await fs.readdir(parent).catch(() => null); - return entries === null || entries.some((entry) => /^docker-[0-9a-f]{64}\.scope$/.test(entry)); -} - /** * Which of the app's processes could still run its code: every one that is * neither gone nor has SIGKILL pending. * @param {string} appName * @param {ContainerRecord[]|null} containers readContainers * @returns {Promise} Their pids; empty when the app can never run - * again; null when none was found but the records could not be read and a - * container outside the slices, which may be the app's, exists. + * again; null when none was found but the records could not be read and + * systemd does not say which containers are outside the slices. */ async function stillRunnable(appName, containers) { const cgroups = await cgroupsOf(appName, containers); - const pids = (await Promise.all(cgroups.map(processesUnder))).flat(); + const known = cgroups ?? await cgroupsOf(appName, []); + const pids = (await Promise.all(known.map(processesUnder))).flat(); const settled = await Promise.all(pids.map(killPendingOrGone)); const runnable = pids.filter((pid, i) => !settled[i]); - if (runnable.length || containers || !await anyContainerOutsideSlices()) return runnable; - return null; + return runnable.length || cgroups ? runnable : null; } /** diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index f94c3e67d2..c8530f7fe7 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -112,9 +112,18 @@ describe('appKill tests', () => { expect(killed).to.deep.equal([]); }); - it('kills the app\'s slice alone when docker\'s records could not be read', async () => { + it('kills the app\'s slice and every container outside the slices when docker\'s records could not be read', async () => { + const appSliceDir = slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + const web = container(1, 'fluxweb_app', ['11']); + const other = container(2, 'fluxother', ['12']); + cgroup('system.slice/fluxos.service', ['1']); + + expect(await appKill.kill('app', null)).to.have.members([appSliceDir, web, other]); + expect(killed).to.have.members([appSliceDir, web, other]); + }); + + it('kills the app\'s slice when docker\'s records could not be read and systemd does not say where system.slice is', async () => { const appSliceDir = slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); - container(1, 'fluxweb_app', ['11']); expect(await appKill.kill('app', null)).to.deep.equal([appSliceDir]); }); @@ -166,10 +175,18 @@ describe('appKill tests', () => { expect(await appKill.stillRunnable('app', null)).to.deep.equal(['21']); }); - it('cannot say the app can no longer run while a container lives outside the slices', async () => { - container(3, 'fluxother_other', ['31']); + it('names what can still run of a container outside the slices, which may be the app\'s', async () => { + container(3, 'fluxweb_other', ['31']); + processWith('31'); - expect(await appKill.stillRunnable('app', null)).to.equal(null); + expect(await appKill.stillRunnable('app', null)).to.deep.equal(['31']); + }); + + it('says the app can no longer run once every container outside the slices has the kill pending', async () => { + container(3, 'fluxweb_other', ['31']); + processWith('31', { sigPnd: '0000000000000100' }); + + expect(await appKill.stillRunnable('app', null)).to.deep.equal([]); }); it('says the app can no longer run when every container lives in a slice', async () => { From 8a314433ff80b60a43a630c67bebea9c5a4116f8 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:15:15 +0100 Subject: [PATCH 260/369] docs(apps): the host-stall unload's doc names the reads at once and the kill outside the slices Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMonitoring/nodeStatusMonitor.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 620193ebf9..a5a486a1dd 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -130,12 +130,13 @@ async function stopApp(name, containers) { * again. Then each app gets its normal removal, which tells the network, within * REMOVAL_DEADLINE_MS; nothing of it runs by then, so the removals take turns. * - * The apps are read from the database and from what is on disk, so an app the - * database cannot list is still found. Docker's records of its containers are - * read once, within STOP_WINDOW_MS: on a disk that does not answer they are not - * waited for further, every app is still stopped and killed through its slice, - * and an app that may have a container outside its slice is reported with - * runnable null, since what of it can run cannot be known. + * The apps are read from the database and from what is on disk, at once, so an + * app the database cannot list is still found. Docker's records of its + * containers are read once, within STOP_WINDOW_MS: on a disk that does not answer + * they are not waited for further, and every app is still stopped and killed + * through its slice, with every container outside the slices (appKill). An app + * is reported with runnable null only when systemd does not say which containers + * are outside the slices. * @param {Function} installedAppsFn * @param {Function} removeAppLocallyFn */ From 60294fd540936716a1864953a4f6c496e60da74f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:17:19 +0100 Subject: [PATCH 261/369] test(harness): 3318 kills a container made before the app slices when a stall holds docker's records Co-Authored-By: Claude Opus 5.5 --- ...-a-container-made-before-the-app-slices.js | 129 ++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js diff --git a/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js b/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js new file mode 100644 index 0000000000..30a15b11ed --- /dev/null +++ b/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js @@ -0,0 +1,129 @@ +// base-image: FLUX_E2E_BASE_TAG +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer } from '../framework/container.js'; +import { waitForUp, waitForAppInstalled } from '../framework/wait.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { buildSeedableTestApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { + DOCKER_ROOT, stallDisk, resumeDisk, readsHeld, thawEveryNode, containerCgroup, processesThatCanRun, +} from '../framework/host-stall.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node upgraded from a release that gave app containers no slice of their own: +// the app's container, made by the release, lives in its runtime's scope under +// system.slice, and only docker's records on disk say which app it is. The node's +// docker disk then stops answering reads, so those records cannot be read, and the +// node goes out of service for the stall. Every container on a node is an app's, +// so it kills every container outside the slices: the app's container among them, +// and the app is reported as unable to run. +// +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; +const READ_INTERVAL_MS = 5 * 1000; +const STUCK_INTERVALS = 6; +const STOP_WINDOW_MS = 10 * 1000; +const REMOVAL_DEADLINE_MS = 45 * 1000; +// what the node takes beyond the window to check and kill every app +const KILL_SLACK_MS = 10 * 1000; +const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; +const UPGRADED = 0; + +const outOfServiceForTheStall = (data) => data.dosState >= 100 && String(data.dosMessage).includes('Host stall'); +const containerOf = (appName) => `flux${appName}_${appName}`; + +// What of a container's cgroup can still run; a cgroup that has gone holds nothing. +async function canRun(container, cgroupDir) { + const exists = await execInContainer(container, `test -d '${cgroupDir}'`); + return exists.exitCode === 0 ? processesThatCanRun(container, cgroupDir) : []; +} + +describe('a stalled host kills a container made before the app slices', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2eprestall${Date.now()}`; + let client; + let container; + let scope; + let runningBefore; + let readsWereHeld; + let trippedAt; + let stopped; + let runnableAfterWindow; + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS the node upgrades from: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + nodeImageTags: { [UPGRADED]: BASE_TAG }, + tickerAutostart: false, + freezableDockerRoot: [UPGRADED], + configOverrides: { + hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS, stopWindowMs: STOP_WINDOW_MS }, + fluxapps: { appRemovalDeadlineMs: REMOVAL_DEADLINE_MS }, + }, + }); + if (BASE_TAG === env.nodeImageTag(1)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG: the suite upgrades from the one to the other'); + await bootAndPeer(env); + client = env.clients[UPGRADED]; + ({ container } = client); + + // an app that writes a log line every 200 ms, so docker writes while its disk stalls + await pushTestApp(appName); + const app = await buildSeedableTestApp({ name: appName, logEveryMs: 200, instances: 1 }); + await seedSpawnerApp(env, app, [UPGRADED]); + await waitForAppInstalled(client, appName, 180000); + await waitForUp(client, appName, 'the app is up on the release'); + scope = await containerCgroup(container, containerOf(appName)); + + await env.upgradeNode(UPGRADED, { tag: env.nodeImageTag(1) }); + await waitForUp(client, appName, 'the app is up after the upgrade'); + runningBefore = await canRun(container, scope); + + const anchor = client.getLastEventId(); + await stallDisk(container, DOCKER_ROOT); + readsWereHeld = await readsHeld(container, `${DOCKER_ROOT}/containers`, { file: 'config.v2.json' }); + const trip = await client.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + // the records' read waits one window, and the stop another + stopped = (await client.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS * 2 + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + runnableAfterWindow = await canRun(container, scope); + }); + + after(async function () { + this.timeout(60000); + if (container) await resumeDisk(container, DOCKER_ROOT).catch(() => {}); + if (client) await thawEveryNode([client]); + await env?.teardown(); + }); + + it('runs the app in a container the release made outside any app slice, still running after the upgrade', function () { + expect(scope, 'fixture: the release put the app\'s container in a slice, so nothing below was tested').to.match(/\/system\.slice\/docker-[0-9a-f]{64}\.scope$/); + expect(runningBefore, 'the app\'s container was not running after the upgrade, so there was nothing to kill').to.not.be.empty; + }); + + it('held every read of docker\'s records of its containers, and went out of service', function () { + expect(readsWereHeld, 'a read of docker\'s container records returned, so the disk was never held').to.equal(true); + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('kills the app\'s container though docker\'s records cannot say it is the app\'s', function () { + expect(stopped, 'the node never said it had stopped its apps').to.not.equal(null); + expect(runnableAfterWindow, 'the app\'s container can still run after the stop window').to.deep.equal([]); + }); + + it('says the app can no longer run', function () { + const reported = stopped?.apps.find((a) => a.app === appName); + expect(reported, 'the node did not name the app among those it stopped').to.not.equal(undefined); + expect(reported.runnable, 'the node reports the app as one it cannot tell can still run, or names its processes').to.deep.equal([]); + }); +}); From 3811e329fc12ec64a88bed02f588ed0f25a3e04b Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:19:32 +0100 Subject: [PATCH 262/369] fix(apps): a host stall kills the containers outside the app slices once, whatever apps it found, when docker's records cannot be read Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appKill.js | 118 +++++++++++++----- .../appMonitoring/nodeStatusMonitor.js | 19 ++- tests/unit/appKill.test.js | 42 +++++-- tests/unit/nodeStatusMonitor.test.js | 50 ++++++++ 4 files changed, 188 insertions(+), 41 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js index 7d227c432a..3ce538db37 100644 --- a/ZelBack/src/services/appLifecycle/appKill.js +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -8,10 +8,12 @@ * read from the runtime's own records on disk (readContainers), which stay * readable while docker does not answer but not while its disk does not. The * records are read once and passed to every call here; null stands for records - * that could not be read, and then every container outside the slices counts as - * each app's: every container on a node is an app's. Where the records are is - * read from docker at boot; until then it is docker's default for the node's - * kind. Where each cgroup is, systemd says (appSlice.controlGroup). + * that could not be read. Then no container outside the slices can be named, so + * those containers are killed and checked together (killOutsideSlices, + * stillRunnableOutsideSlices), and an app can be said to be unable to run only + * once none of them can. Where the records are is read from docker at boot; until + * then it is docker's default for the node's kind. Where each cgroup is, systemd + * says (appSlice.controlGroup). * * A process held in the kernel is not removed by a kill: it keeps the kill * pending and dies the moment the kernel lets it go, without running the app's @@ -152,41 +154,74 @@ async function scopesOutsideSlices() { } /** - * Every cgroup that holds a container of the app and exists now: its slice, and - * the scope of each of its containers the records name; with no records, the - * scope of every container outside the slices. - * @param {string} appName - * @param {ContainerRecord[]|null} containers readContainers - * @returns {Promise} Null when the records could not be read and - * systemd does not say which containers are outside the slices. + * The cgroups of the named units that exist now, each once: a scope inside a slice + * also named is read and killed with the slice. + * @param {string[]} units + * @returns {Promise} */ -async function cgroupsOf(appName, containers) { - const unit = appSlice.sliceName(appName); - const scopes = containers ? recordsOf(appName, containers).map(({ id }) => `docker-${id}.scope`) : await scopesOutsideSlices(); - if (scopes === null) return null; - const units = [...(unit ? [unit] : []), ...scopes]; +async function existingCgroups(units) { const found = (await Promise.all(units.map(appSlice.controlGroup))).filter(Boolean); - // a scope inside the app's slice is read and killed with the slice const outermost = [...new Set(found)].filter((dir, _, all) => !all.some((other) => dir.startsWith(`${other}/`))); const present = await Promise.all(outermost.map((dir) => fs.access(path.join(dir, 'cgroup.procs')).then(() => dir, () => null))); return present.filter(Boolean); } /** - * Send SIGKILL to every process of the app. + * Every cgroup that holds a container of the app and exists now: its slice, and + * the scope of each of its containers the records name. * @param {string} appName * @param {ContainerRecord[]|null} containers readContainers - * @returns {Promise} The cgroups it was sent to. + * @returns {Promise} */ -async function kill(appName, containers) { - const cgroups = await cgroupsOf(appName, containers) - ?? await cgroupsOf(appName, []); +async function cgroupsOf(appName, containers) { + const unit = appSlice.sliceName(appName); + const scopes = recordsOf(appName, containers).map(({ id }) => `docker-${id}.scope`); + return existingCgroups([...(unit ? [unit] : []), ...scopes]); +} + +/** + * The cgroup of every container outside the app slices that exists now. + * @returns {Promise} null when systemd does not answer. + */ +async function outsideSliceCgroups() { + const scopes = await scopesOutsideSlices(); + return scopes === null ? null : existingCgroups(scopes); +} + +/** + * Send SIGKILL to every process in each cgroup. + * @param {string} owner What the cgroups hold, for the log. + * @param {string[]} cgroups + */ +async function killIn(owner, cgroups) { // eslint-disable-next-line no-restricted-syntax for (const dir of cgroups) { // eslint-disable-next-line no-await-in-loop const { error } = await serviceHelper.runCommand('sh', { runAsRoot: true, params: ['-c', 'echo 1 > "$1"', 'sh', path.join(dir, 'cgroup.kill')] }); - if (error) log.error(`appKill - ${appName}: the kill could not be sent to ${dir}: ${error.message}`); + if (error) log.error(`appKill - ${owner}: the kill could not be sent to ${dir}: ${error.message}`); } +} + +/** + * Send SIGKILL to every process of the app. + * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers + * @returns {Promise} The cgroups it was sent to. + */ +async function kill(appName, containers) { + const cgroups = await cgroupsOf(appName, containers); + await killIn(appName, cgroups); + return cgroups; +} + +/** + * Send SIGKILL to every process of every container outside the app slices. + * @returns {Promise} The cgroups it was sent to; null when systemd + * does not say which containers are outside the slices. + */ +async function killOutsideSlices() { + const cgroups = await outsideSliceCgroups(); + if (cgroups !== null) await killIn('containers outside the app slices', cgroups); return cgroups; } @@ -217,22 +252,41 @@ async function processesUnder(dir) { return [...own, ...below.flat()]; } +/** + * Every process in the cgroups that is neither gone nor has SIGKILL pending. + * @param {string[]} cgroups + * @returns {Promise} Their pids. + */ +async function runnableIn(cgroups) { + const pids = (await Promise.all(cgroups.map(processesUnder))).flat(); + const settled = await Promise.all(pids.map(killPendingOrGone)); + return pids.filter((pid, i) => !settled[i]); +} + +/** + * Which processes of the containers outside the app slices could still run. + * @returns {Promise} Their pids; null when systemd does not say + * which containers are outside the slices. + */ +async function stillRunnableOutsideSlices() { + const cgroups = await outsideSliceCgroups(); + return cgroups === null ? null : runnableIn(cgroups); +} + /** * Which of the app's processes could still run its code: every one that is * neither gone nor has SIGKILL pending. * @param {string} appName * @param {ContainerRecord[]|null} containers readContainers * @returns {Promise} Their pids; empty when the app can never run - * again; null when none was found but the records could not be read and - * systemd does not say which containers are outside the slices. + * again; null when none was found, the records could not be read, and a + * container outside the slices, which may be the app's, can still run or + * cannot be found. */ async function stillRunnable(appName, containers) { - const cgroups = await cgroupsOf(appName, containers); - const known = cgroups ?? await cgroupsOf(appName, []); - const pids = (await Promise.all(known.map(processesUnder))).flat(); - const settled = await Promise.all(pids.map(killPendingOrGone)); - const runnable = pids.filter((pid, i) => !settled[i]); - return runnable.length || cgroups ? runnable : null; + const runnable = await runnableIn(await cgroupsOf(appName, containers)); + if (runnable.length || containers) return runnable; + return (await stillRunnableOutsideSlices())?.length === 0 ? [] : null; } /** @@ -260,7 +314,9 @@ module.exports = { readContainers, containersOf, kill, + killOutsideSlices, stillRunnable, + stillRunnableOutsideSlices, appsOnDisk, usePaths, reset, diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index a5a486a1dd..99ebb8f3b4 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -151,11 +151,23 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { ]); const installed = listed?.status === 'success' ? listed.data.map((app) => app.name) : []; const containers = Array.isArray(read) ? read : null; - if (!containers) log.error(`monitorNodeStatus - host stall: docker's container records could not be read (${read === TIMED_OUT ? 'no answer' : 'refused'}); containers outside the app slices cannot be found`); + if (!containers) log.error(`monitorNodeStatus - host stall: docker's container records could not be read (${read === TIMED_OUT ? 'no answer' : 'refused'}); every container outside the app slices is killed`); const names = [...new Set([...installed, ...await appKill.appsOnDisk(containers)])]; log.warn(`monitorNodeStatus - host stall: stopping ${names.length} applications within one ${STOP_WINDOW_MS / 1000}s window`); await withinDeadline(Promise.allSettled(names.map((name) => stopApp(name, containers))), STOP_WINDOW_MS); + // With no records, a container outside the slices cannot be named, whatever apps + // were found: every one is killed and checked, once. + let outsideSlices; + if (!containers) { + const killedIn = await appKill.killOutsideSlices(); + const runnable = await appKill.stillRunnableOutsideSlices(); + outsideSlices = { containers: killedIn?.length ?? null, runnable }; + if (runnable === null) log.error('monitorNodeStatus - host stall: which containers are outside the app slices cannot be known'); + else if (runnable.length) log.error(`monitorNodeStatus - host stall: processes ${runnable.join(', ')} of containers outside the app slices can still run`); + else log.warn(`monitorNodeStatus - host stall: nothing of the ${killedIn.length} containers outside the app slices can run`); + } + const stopped = new Map(); // eslint-disable-next-line no-restricted-syntax for (const name of names) { @@ -178,9 +190,12 @@ async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { stopped.set(name, { killed, runnable }); } // runnable: the processes of each app that can still run its code, empty when - // none can, null when that cannot be known + // none can, null when that cannot be known; outsideSlices, when the records could + // not be read: how many containers outside the slices were killed, and which of + // their processes can still run fluxEventBus.publish('hostStall:appsStopped', { apps: names.map((name) => ({ app: name, ...stopped.get(name) })), + ...(outsideSlices ? { outsideSlices } : {}), }); const canBroadcast = nodeConfirmationService.canSendMessages(); diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js index c8530f7fe7..3c0b1b9d51 100644 --- a/tests/unit/appKill.test.js +++ b/tests/unit/appKill.test.js @@ -112,20 +112,30 @@ describe('appKill tests', () => { expect(killed).to.deep.equal([]); }); - it('kills the app\'s slice and every container outside the slices when docker\'s records could not be read', async () => { + it('kills the app\'s slice alone when docker\'s records could not be read', async () => { const appSliceDir = slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + container(1, 'fluxweb_app', ['11']); + + expect(await appKill.kill('app', null)).to.deep.equal([appSliceDir]); + }); + }); + + describe('killOutsideSlices', () => { + it('kills every container outside the app slices, and no other unit', async () => { + slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); const web = container(1, 'fluxweb_app', ['11']); const other = container(2, 'fluxother', ['12']); cgroup('system.slice/fluxos.service', ['1']); - expect(await appKill.kill('app', null)).to.have.members([appSliceDir, web, other]); - expect(killed).to.have.members([appSliceDir, web, other]); + expect(await appKill.killOutsideSlices()).to.have.members([web, other]); + expect(killed).to.have.members([web, other]); }); - it('kills the app\'s slice when docker\'s records could not be read and systemd does not say where system.slice is', async () => { - const appSliceDir = slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + it('kills nothing and says so when systemd does not say where system.slice is', async () => { + slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); - expect(await appKill.kill('app', null)).to.deep.equal([appSliceDir]); + expect(await appKill.killOutsideSlices()).to.equal(null); + expect(killed).to.deep.equal([]); }); it('passes over a container record it cannot read', async () => { @@ -175,11 +185,11 @@ describe('appKill tests', () => { expect(await appKill.stillRunnable('app', null)).to.deep.equal(['21']); }); - it('names what can still run of a container outside the slices, which may be the app\'s', async () => { + it('cannot say the app can no longer run while a container outside the slices, which may be the app\'s, can run', async () => { container(3, 'fluxweb_other', ['31']); processWith('31'); - expect(await appKill.stillRunnable('app', null)).to.deep.equal(['31']); + expect(await appKill.stillRunnable('app', null)).to.equal(null); }); it('says the app can no longer run once every container outside the slices has the kill pending', async () => { @@ -201,6 +211,22 @@ describe('appKill tests', () => { }); }); + describe('stillRunnableOutsideSlices', () => { + it('names what can still run of every container outside the app slices', async () => { + container(1, 'fluxweb_a', ['11']); + container(2, 'fluxweb_b', ['12', '13']); + cgroup('system.slice/fluxos.service', ['1']); + ['1', '11', '12'].forEach((pid) => processWith(pid)); + processWith('13', { sigPnd: '0000000000000100' }); + + expect(await appKill.stillRunnableOutsideSlices()).to.have.members(['11', '12']); + }); + + it('cannot say when systemd does not say where system.slice is', async () => { + expect(await appKill.stillRunnableOutsideSlices()).to.equal(null); + }); + }); + describe('readContainers', () => { it('answers null when docker\'s records cannot be listed', async () => { fs.rmSync(path.join(dockerRoot, 'containers'), { recursive: true }); diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index 7eb6cb0be6..041d28841f 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -164,6 +164,8 @@ describe('nodeStatusMonitor tests', () => { containersOf: sinon.stub().callsFake((name) => [`flux${name}`]), stillRunnable: sinon.stub().resolves([]), kill: sinon.stub().resolves([]), + killOutsideSlices: sinon.stub().resolves([]), + stillRunnableOutsideSlices: sinon.stub().resolves([]), }; dockerService = { appDockerStop: sinon.stub().resolves('stopped') }; monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { @@ -384,6 +386,54 @@ describe('nodeStatusMonitor tests', () => { await unload; }); + describe('when docker\'s records cannot be read', () => { + beforeEach(() => { + appKill.readContainers.resolves(null); + }); + + // A disk the database and docker share stalls both: no app is listed, and the + // containers made before the slices are found by no name. + it('kills every container outside the slices though no app is found, and says what can still run of them', async () => { + appKill.killOutsideSlices.resolves(['/sys/fs/cgroup/system.slice/docker-1.scope']); + appKill.stillRunnableOutsideSlices.resolves([]); + + const unload = monitor.unloadForHostStall(sinon.stub().callsFake(never), sinon.stub().resolves('removed')); + await clock.tickAsync(DEADLINE + STOP_WINDOW); + await unload; + + sinon.assert.calledOnce(appKill.killOutsideSlices); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { apps: [], outsideSlices: { containers: 1, runnable: [] } }); + }); + + it('kills them once whatever the number of apps, after the stop window', async () => { + dockerService.appDockerStop.callsFake(never); + + const unload = monitor.unloadForHostStall(installed('a', 'b', 'c'), sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW - 1); + sinon.assert.notCalled(appKill.killOutsideSlices); + await clock.tickAsync(1); + await unload; + + sinon.assert.calledOnce(appKill.killOutsideSlices); + }); + + it('names what of them can still run, or that it cannot be known', async () => { + appKill.stillRunnableOutsideSlices.onFirstCall().resolves(['77']); + + await monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + + sinon.assert.calledWithMatch(logStub.error, /processes 77 of containers outside the app slices can still run/); + sinon.assert.calledWithMatch(bus.publish, 'hostStall:appsStopped', { outsideSlices: { runnable: ['77'] } }); + }); + }); + + it('kills nothing outside the slices when docker\'s records name every container', async () => { + await monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + + sinon.assert.notCalled(appKill.killOutsideSlices); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { apps: [{ app: 'a', killed: false, runnable: [] }] }); + }); + it('names a refusal to list docker\'s records', async () => { appKill.readContainers.resolves(null); From 54873da620dc7959140927a6a12d9abd3057704f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:20:12 +0100 Subject: [PATCH 263/369] test(harness): 3318 kills a container made before the app slices when the database names no app either Co-Authored-By: Claude Opus 5.5 --- ...-a-container-made-before-the-app-slices.js | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js b/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js index 30a15b11ed..d458b59b61 100644 --- a/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js +++ b/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js @@ -10,15 +10,17 @@ import { pushTestApp } from '../framework/registry-helper.js'; import { DOCKER_ROOT, stallDisk, resumeDisk, readsHeld, thawEveryNode, containerCgroup, processesThatCanRun, } from '../framework/host-stall.js'; +import { failCommands } from '../framework/mongo-control.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // A node upgraded from a release that gave app containers no slice of their own: // the app's container, made by the release, lives in its runtime's scope under // system.slice, and only docker's records on disk say which app it is. The node's -// docker disk then stops answering reads, so those records cannot be read, and the -// node goes out of service for the stall. Every container on a node is an app's, -// so it kills every container outside the slices: the app's container among them, -// and the app is reported as unable to run. +// docker disk then stops answering reads, so those records cannot be read, and its +// database refuses the read of its installed apps, as one on the same disk does, so +// the node finds no app by name. It goes out of service for the stall, and kills +// every container outside the slices all the same: the app's container among them, +// and nothing of those containers is reported as able to run. // // Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. @@ -53,6 +55,7 @@ describe('a stalled host kills a container made before the app slices', function let trippedAt; let stopped; let runnableAfterWindow; + let releaseDatabase = null; before(async function () { this.timeout(900000); @@ -88,6 +91,7 @@ describe('a stalled host kills a container made before the app slices', function runningBefore = await canRun(container, scope); const anchor = client.getLastEventId(); + releaseDatabase = await failCommands(UPGRADED + 1, 'localzelapps', 'zelappsinformation', { commands: ['find'] }); await stallDisk(container, DOCKER_ROOT); readsWereHeld = await readsHeld(container, `${DOCKER_ROOT}/containers`, { file: 'config.v2.json' }); const trip = await client.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) @@ -101,6 +105,7 @@ describe('a stalled host kills a container made before the app slices', function after(async function () { this.timeout(60000); + await releaseDatabase?.().catch(() => {}); if (container) await resumeDisk(container, DOCKER_ROOT).catch(() => {}); if (client) await thawEveryNode([client]); await env?.teardown(); @@ -116,14 +121,15 @@ describe('a stalled host kills a container made before the app slices', function expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); }); - it('kills the app\'s container though docker\'s records cannot say it is the app\'s', function () { + it('kills the app\'s container though neither docker\'s records nor the database name the app', function () { expect(stopped, 'the node never said it had stopped its apps').to.not.equal(null); + expect(stopped.apps.map((a) => a.app), 'fixture: the node found the app by name, so nothing below was tested').to.not.include(appName); expect(runnableAfterWindow, 'the app\'s container can still run after the stop window').to.deep.equal([]); }); - it('says the app can no longer run', function () { - const reported = stopped?.apps.find((a) => a.app === appName); - expect(reported, 'the node did not name the app among those it stopped').to.not.equal(undefined); - expect(reported.runnable, 'the node reports the app as one it cannot tell can still run, or names its processes').to.deep.equal([]); + it('says nothing of the containers outside the app slices can run', function () { + expect(stopped?.outsideSlices, 'the node did not report the containers outside the app slices').to.not.equal(undefined); + expect(stopped.outsideSlices.containers, 'the node killed no container outside the app slices').to.be.at.least(1); + expect(stopped.outsideSlices.runnable, 'the node reports them as ones it cannot tell can still run, or names their processes').to.deep.equal([]); }); }); From 46211dc1f723c8ef91412942db894a5cd2e1456d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:40:20 +0100 Subject: [PATCH 264/369] fix(apps): boot does not wait for a host stall's state write, and the writes land one at a time Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/hostStallService.js | 56 +++++++++++++++++------- tests/unit/hostStallService.test.js | 56 ++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 16 deletions(-) diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js index f40d306696..b2d49af3b4 100644 --- a/ZelBack/src/services/hostStallService.js +++ b/ZelBack/src/services/hostStallService.js @@ -68,6 +68,8 @@ let paths = { ...DEFAULT_PATHS }; const NOTHING_KEPT = JSON.stringify({ trip: null, threads: [] }); /** What the state file holds, as this process last wrote or found it; null when unknown. */ let keptText = NOTHING_KEPT; +/** What the state file is to hold once the write asked for last has landed; null when none is waiting. */ +let queuedText = null; /** * @param {{procRoot?: string, stateFile?: string}} options @@ -221,14 +223,11 @@ async function makeStateDirectory(dir) { } /** - * Keep the trip and every thread's count, so a FluxOS restart before the next - * reboot carries both on. Nothing is written when the file already holds them. + * Write the state file, once every write asked for before has ended. + * @param {string} text + * @param {string} stateFile The file as it was when the write was asked for. */ -async function keepState() { - const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; - const text = JSON.stringify({ trip, threads: [...tracked] }); - if (text === keptText) return; - const { stateFile } = paths; +async function writeState(text, stateFile) { const staged = `${stateFile}.tmp`; try { await makeStateDirectory(path.dirname(stateFile)); @@ -237,9 +236,26 @@ async function keepState() { keptText = text; } catch (error) { log.error(`hostStall - the state could not be kept: ${error.message}`); + } finally { + if (queuedText === text) queuedText = null; } } +/** + * Keep the trip and every thread's count, so a FluxOS restart before the next + * reboot carries both on. Nothing is written when the file already holds them, or + * a write waiting to land does. Writes land one at a time and in order, and no + * reading waits for one. + */ +function keepState() { + const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; + const text = JSON.stringify({ trip, threads: [...tracked] }); + if (text === (queuedText ?? keptText)) return; + queuedText = text; + const { stateFile } = paths; + keeping = keeping.then(() => writeState(text, stateFile)); +} + /** Tell every listener the node is out of service for a host stall. */ function announceTrip() { tripListeners.forEach((listener) => listener()); @@ -290,12 +306,8 @@ async function act() { } } -/** - * Take one reading and act on it. - * @param {object} [options] Paths, as usePaths takes them. - */ -async function readOnce(options = {}) { - usePaths(options); +/** Take one reading, act on it, and ask for what it found to be kept. */ +async function takeReading() { const startedIn = generation; const nowMs = await readUptimeMs(paths.procRoot); if (nowMs === null) { @@ -306,13 +318,23 @@ async function readOnce(options = {}) { if (startedIn !== generation) return; tracked = carryForward(threads, nowMs); await act(); - keeping = keepState(); + keepState(); +} + +/** + * Take one reading and act on it. + * @param {object} [options] Paths, as usePaths takes them. + * @returns {Promise} Resolves once what it found is kept. + */ +async function readOnce(options = {}) { + usePaths(options); + await takeReading(); await keeping; } async function readAndReschedule() { try { - await readOnce(); + await takeReading(); } catch (error) { log.error(`hostStall - reading the dependency threads failed: ${error.message}`); } @@ -323,7 +345,8 @@ async function readAndReschedule() { * Restore what an earlier FluxOS in this boot kept, then read now, and every * READ_INTERVAL_MS after each reading ends. * @param {object} [options] Paths, as usePaths takes them. - * @returns {Promise} Resolves once the kept state is restored and the first reading taken. + * @returns {Promise} Resolves once the kept state is restored and the first + * reading has acted; the write of what it found goes on without it. */ async function start(options = {}) { if (running) return; @@ -345,6 +368,7 @@ function stop() { tracked = new Map(); tripReason = null; keptText = NOTHING_KEPT; + queuedText = null; paths = { ...DEFAULT_PATHS }; tripListeners.length = 0; return keeping; diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index e977a9ef6d..93d85fcbb2 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -422,6 +422,62 @@ describe('hostStallService tests', () => { }); }); + describe('a write that does not end', () => { + let releases; + let holding; + beforeEach(() => { + releases = []; + holding = true; + const writeFile = fs.promises.writeFile.bind(fs.promises); + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (holding && String(file).startsWith(paths.stateFile) + ? new Promise((resolve) => { releases.push(() => resolve(writeFile(file, ...rest))); }) + : writeFile(file, ...rest))); + }); + + it('holds up no start: a start returns once its first reading has acted', async () => { + holding = false; + stuckMongod(); + await readTimes(stuckIntervals); + await hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + passMs(readIntervalMs); + holding = true; + + await hostStallService.start(paths); + + expect(outOfService(), 'the start\'s reading did not act').to.equal(true); + for (let i = 0; i < 100 && !releases.length; i += 1) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop + expect(releases.length, 'fixture: the start asked for no write').to.equal(1); + const stopped = hostStallService.stop(); + releases.forEach((release) => release()); + await stopped; + }); + + it('lands the writes one at a time and in order, the last one asked for last', async () => { + const until = async (done) => { + for (let i = 0; i < 2000 && !done(); i += 1) await new Promise((resolve) => { setTimeout(resolve, 1); }); // eslint-disable-line no-await-in-loop + }; + stuckMongod(); + passMs(readIntervalMs); + const first = hostStallService.readOnce(paths); + await until(() => releases.length === 1); + expect(releases.length, 'fixture: the first reading asked for no write').to.equal(1); + + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 41 }]); + passMs(readIntervalMs); + const second = hostStallService.readOnce(paths); + await new Promise((resolve) => { setTimeout(resolve, 50); }); + expect(releases.length, 'the second write started beside the first').to.equal(1); + + releases[0](); + await until(() => releases.length === 2); + releases[1](); + await Promise.all([first, second]); + + expect(kept().threads).to.deep.equal([]); + }); + }); + describe('a FluxOS that may not make the state directory', () => { let stateDir; beforeEach(() => { From bea0e5ac3f6a7955d51c68390e800c118afe88f6 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:40:58 +0100 Subject: [PATCH 265/369] test(harness): 3319 boots past the host-stall start while the state write does not end Co-Authored-By: Claude Opus 5.5 --- ...s-not-wait-for-a-host-stall-state-write.js | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js diff --git a/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js b/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js new file mode 100644 index 0000000000..be351f03dc --- /dev/null +++ b/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js @@ -0,0 +1,68 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A FluxOS that starts while the directory its host-stall state is kept in takes no +// writes: the state there cannot be read, so the start's first reading writes it, +// and that write does not end. The start returns once the reading has acted, and +// boot goes on to its next step. +// +// The node's /run/fluxos is a filesystem of its own, frozen, so a read of it answers +// and a write to it is held in the kernel. + +const STATE_DIR = '/run/fluxos'; +const STATE_FILE = `${STATE_DIR}/host-stall.json`; +const IMAGE = '/tmp/flux-e2e-run-fluxos.img'; +// Boot's line after the host-stall service has started (serviceManager). +const NEXT_STEP = 'MongoDB connected'; + +describe('a boot does not wait for a host stall\'s state write', function () { + let env; + dumpLogsOnFailure(() => env); + let client; + let writesHeld; + let bootLinesBefore; + + const sh = async (command) => { + const r = await execInContainer(client.container, command); + if (r.exitCode !== 0) throw new Error(`fixture: ${command}: ${r.output}`); + return r.stdout; + }; + + before(async function () { + this.timeout(300000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + [client] = env.clients; + await waitFor(() => env.nodeLogCount(0, NEXT_STEP) >= 1, { timeout: 120000, interval: 1000, label: 'the node booted' }); + + await crashFluxos(client.container, { hold: true }); + await sh(`rm -rf ${STATE_DIR} && mkdir -p ${STATE_DIR} && dd if=/dev/zero of=${IMAGE} bs=1M count=16 status=none` + + ` && mke2fs -q -t ext4 ${IMAGE} && mount -o loop ${IMAGE} ${STATE_DIR}` + + ` && printf '{ not json' > ${STATE_FILE} && sync -f ${STATE_FILE} && fsfreeze -f ${STATE_DIR}`); + const probe = await execInContainer(client.container, `timeout 3 sh -c 'echo x > ${STATE_DIR}/probe'`); + writesHeld = probe.exitCode === 124; + + bootLinesBefore = env.nodeLogCount(0, NEXT_STEP); + await releaseFluxos(client.container); + }); + + after(async function () { + this.timeout(60000); + if (client) await execInContainer(client.container, `fsfreeze -u ${STATE_DIR}; umount -l ${STATE_DIR}; rm -f ${IMAGE}`).catch(() => {}); + await env?.teardown(); + }); + + it('holds every write to the state directory', function () { + expect(writesHeld, 'a write to the state directory returned, so nothing below was tested').to.equal(true); + }); + + it('goes on to the next step of boot while the state write does not end', async function () { + this.timeout(120000); + await waitFor(() => env.nodeLogCount(0, NEXT_STEP) > bootLinesBefore, { + timeout: 60000, interval: 1000, label: 'boot waited for the host stall\'s state write', + }); + }); +}); From 329bbdafeeedf157135410590e6a476f291d16e3 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:47:14 +0100 Subject: [PATCH 266/369] test(harness): 3319's canary holds its probe write in the background Co-Authored-By: Claude Opus 5.5 --- ...9-a-boot-does-not-wait-for-a-host-stall-state-write.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js b/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js index be351f03dc..f45a625fc1 100644 --- a/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js +++ b/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js @@ -42,8 +42,12 @@ describe('a boot does not wait for a host stall\'s state write', function () { await sh(`rm -rf ${STATE_DIR} && mkdir -p ${STATE_DIR} && dd if=/dev/zero of=${IMAGE} bs=1M count=16 status=none` + ` && mke2fs -q -t ext4 ${IMAGE} && mount -o loop ${IMAGE} ${STATE_DIR}` + ` && printf '{ not json' > ${STATE_FILE} && sync -f ${STATE_FILE} && fsfreeze -f ${STATE_DIR}`); - const probe = await execInContainer(client.container, `timeout 3 sh -c 'echo x > ${STATE_DIR}/probe'`); - writesHeld = probe.exitCode === 124; + // a write held in the kernel cannot be killed: it runs in the background, and is + // held while its marker has not appeared + const marker = `/tmp/state-write-returned-${Date.now()}`; + const probe = await execInContainer(client.container, + `setsid sh -c 'echo x > ${STATE_DIR}/probe && touch ${marker}' /dev/null 2>&1 & sleep 3; test ! -e ${marker}`); + writesHeld = probe.exitCode === 0; bootLinesBefore = env.nodeLogCount(0, NEXT_STEP); await releaseFluxos(client.container); From 114b9f345cc2bf0a883e3724dbddd96d76bade52 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 16:03:42 +0100 Subject: [PATCH 267/369] test(apps): the held host-stall write tests wait for the write by time and release every held write after each test Co-Authored-By: Claude Opus 5.5 --- tests/unit/hostStallService.test.js | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 93d85fcbb2..2febd02467 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -425,6 +425,15 @@ describe('hostStallService tests', () => { describe('a write that does not end', () => { let releases; let holding; + const until = async (done) => { + for (let i = 0; i < 2000 && !done(); i += 1) await new Promise((resolve) => { setTimeout(resolve, 1); }); // eslint-disable-line no-await-in-loop + }; + // A write left held would hold the stop after each test, and the node out of + // service, for every test after it. + afterEach(() => { + holding = false; + releases.forEach((release) => release()); + }); beforeEach(() => { releases = []; holding = true; @@ -446,17 +455,11 @@ describe('hostStallService tests', () => { await hostStallService.start(paths); expect(outOfService(), 'the start\'s reading did not act').to.equal(true); - for (let i = 0; i < 100 && !releases.length; i += 1) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop + await until(() => releases.length > 0); expect(releases.length, 'fixture: the start asked for no write').to.equal(1); - const stopped = hostStallService.stop(); - releases.forEach((release) => release()); - await stopped; }); it('lands the writes one at a time and in order, the last one asked for last', async () => { - const until = async (done) => { - for (let i = 0; i < 2000 && !done(); i += 1) await new Promise((resolve) => { setTimeout(resolve, 1); }); // eslint-disable-line no-await-in-loop - }; stuckMongod(); passMs(readIntervalMs); const first = hostStallService.readOnce(paths); From e2d0bcb92c94643045f087d14a97ecd954b55d60 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 16:45:41 +0100 Subject: [PATCH 268/369] fix(syncthing): a peer stays in a folder while it is connected and shares it A location expires when its node stops announcing, and a node whose FluxOS has stopped answering stops announcing while its syncthing and the app carry on. A device the locations no longer name keeps its place in the folder while syncthing reports it connected and sharing the folder (remoteState 'valid'), or while that cannot be read, and the device sweep keeps it. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/syncthingMonitor.js | 34 ++++++++++++++ tests/unit/syncthingMonitor.test.js | 46 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index a4948fce42..1dc176a551 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -249,6 +249,31 @@ async function readAppLocation(appName) { return results || []; } +/** + * The devices of a folder that its app's locations no longer name but that are + * still connected and sharing it. + * + * A location expires when its node stops announcing, and a node can stop + * announcing while its syncthing carries on - a FluxOS that has stopped answering + * leaves the app and syncthing running. Such a holder may still be the app's + * primary, writing, so it keeps its place until its connection closes or it stops + * sharing the folder. A device whose state cannot be read keeps its place too. + * @param {string} folderId + * @param {Array<{deviceID: string}>} configured The folder's devices in syncthing now. + * @param {Array<{deviceID: string}>} named The devices the app's locations name. + * @returns {Promise} Device IDs to keep in the folder. + */ +async function connectedUnnamedDevices(folderId, configured, named) { + const unnamed = (configured || []) + .map((device) => device.deviceID) + .filter((deviceID) => !named.some((device) => device.deviceID === deviceID)); + const kept = await Promise.all(unnamed.map(async (deviceID) => { + const completion = await syncthingService.getDbCompletion({ folder: folderId, device: deviceID }).catch(() => null); + return !completion || completion.remoteState === 'valid' ? deviceID : null; + })); + return kept.filter(Boolean); +} + /** * An app's location records, or none when they cannot be read. * @param {string} appName @@ -357,6 +382,15 @@ async function processContainerData(params) { allDevices, ); + if (syncFolder) { + const kept = await connectedUnnamedDevices(id, syncFolder.devices, devices); + kept.forEach((deviceID) => { + log.info(`processContainerData - ${appId}: keeping device ${deviceID.substring(0, 7)}, connected and sharing the folder though no location names it`); + devices.push({ deviceID }); + if (!devicesIds.includes(deviceID)) devicesIds.push(deviceID); + }); + } + // Create base folder configuration const syncthingFolder = createSyncthingFolderConfig(id, label, folder, devices); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index 7df849ac91..232eecef5a 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -36,6 +36,7 @@ const syncthingServiceMock = { systemRestart: sinon.stub().resolves(), systemResume: sinon.stub().resolves(), getDbStatus: sinon.stub(), + getDbCompletion: sinon.stub().resolves({ remoteState: 'notSharing' }), }; const syncthingFolderStateMachineMock = { @@ -1844,6 +1845,51 @@ describe('syncthingMonitor tests', () => { // because a pass failed to reach the component that owns it. Deleting a live // folder drops syncthing's whole record of the app: the index, the peer // devices and, on the next pass, any mount-safety demotion standing on it. + // A location expires when its node stops announcing, which a node whose FluxOS has + // stopped answering does while its syncthing, and the app, carry on. + describe('a peer no location names any more', () => { + function folderSharedWithPeer() { + mockState.backupInProgress = []; + mockState.restoreInProgress = []; + syncthingMonitorHelpersMock.buildDeviceConfiguration.callsFake(async () => [{ deviceID: 'DEVICE-ID' }]); + syncthingMonitorHelpersMock.getContainerDataFlags.returns('g'); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingServiceMock.getDeviceId.resolves('DEVICE-ID'); + fluxNetworkHelperMock.getLocalSocketAddress.resolves('10.0.0.1:16127'); + mockInstalledAppsFn.resolves({ status: 'success', data: [{ name: 'testapp', version: 3, containerData: 'g:/appdata' }] }); + syncthingServiceMock.getConfigFolders.resolves([ + { id: 'testapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }, { deviceID: 'PEER-DEVICE' }] }, + ]); + syncthingServiceMock.getConfigDevices.resolves([{ deviceID: 'PEER-DEVICE' }]); + syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + syncthingServiceMock.adjustConfigDevices.resolves({ status: 'success', data: {} }); + } + + afterEach(() => { + syncthingServiceMock.getDbCompletion.reset(); + syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'notSharing' }); + }); + + it('keeps its device while it is connected and shares the folder', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getDbCompletion.withArgs({ folder: 'testapp', device: 'PEER-DEVICE' }).resolves({ remoteState: 'valid' }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + + it('removes its device once it no longer shares the folder', async () => { + folderSharedWithPeer(); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + }); + describe('unused-folder cleanup', () => { const syncingApp = { name: 'testapp', version: 3, containerData: 'g:/appdata' }; From 93293f98f4a145c0b49cf2ebc553543a8b70e133 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 16:45:41 +0100 Subject: [PATCH 269/369] fix(syncthing): a local change no other device holds is kept, not reverted syncthing reverts a receive-only change whose cluster version is this node's own by deleting it, and a directory that is not empty cannot be deleted, so that revert fails on every pass and the standby never becomes ready. Before reverting, each local change is checked (db/file, global localFlags & 8): a folder holding a change no other device holds is not reverted, publishes syncthing:localChangesKept, and is promoted with that change. A folder whose changes cannot be checked is neither reverted nor promoted that pass. Co-Authored-By: Claude Opus 5.5 --- .../syncthingFolderStateMachine.js | 65 ++++++++++++++--- ZelBack/src/services/syncthingService.js | 12 +++ test-infra/runner/framework/node-client.js | 1 + .../unit/syncthingFolderStateMachine.test.js | 73 +++++++++++++++++++ 4 files changed, 142 insertions(+), 9 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 8bd82b9221..90715f3f44 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1288,6 +1288,39 @@ async function nudgeFolderDevices(folderId) { } } +// syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const LOCAL_CHANGE_PAGE_SIZE = 1000; + +/** + * Whether any of a receive-only folder's local changes is held by no other device: + * the cluster's version of it is this node's own change. syncthing reverts such an + * item by deleting it, and a directory that is not empty cannot be deleted, so that + * revert fails on every pass. + * @param {string} appId - The Syncthing folder ID + * @returns {Promise} null when the changes could not be read + */ +async function localChangeHeldOnlyHere(appId) { + try { + for (let page = 1; ; page += 1) { + // eslint-disable-next-line no-await-in-loop + const batch = (await syncthingService.getDbLocalChanged(appId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; + if (!Array.isArray(batch)) return page > 1 ? false : null; + // eslint-disable-next-line no-restricted-syntax + for (const item of batch) { + // eslint-disable-next-line no-await-in-loop + const { global } = await syncthingService.getDbFile(appId, item.name); + // eslint-disable-next-line no-bitwise + if ((global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) return true; + } + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return false; + } + } catch (error) { + log.warn(`localChangeHeldOnlyHere - ${appId}: ${error.message}`); + return null; + } +} + /** * Reverts a synced receiveonly folder's local changes to the cluster's version. * @@ -1297,24 +1330,37 @@ async function nudgeFolderDevices(folderId) { * version and deletes local-only items that are not ignored, so it runs only * against a complete global index (isSynced). * + * A change no other device holds is this node's data, and db/revert would delete + * it. A folder holding one is not reverted, and its changes do not hold the node + * back from taking over: promoted, it publishes them. + * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status - * @returns {Promise} True when the folder had local changes and a revert was asked for + * @returns {Promise} True when the folder has local changes it does not + * keep: a revert was asked for, or which changes other devices hold could not be read */ async function revertLocalChangesIfSynced(appId, syncStatus) { if (!syncStatus?.isSynced) return false; const changed = (syncStatus.receiveOnlyChangedFiles || 0) + (syncStatus.receiveOnlyChangedDirectories || 0); if (changed === 0) return false; - log.warn(`revertLocalChangesIfSynced - ${appId} is synced but the receive-only folder has ${changed} locally changed item(s) (${syncStatus.receiveOnlyChangedFiles || 0} files, ${syncStatus.receiveOnlyChangedDirectories || 0} directories); reverting to the cluster's version`); + const files = syncStatus.receiveOnlyChangedFiles || 0; + const directories = syncStatus.receiveOnlyChangedDirectories || 0; + const heldOnlyHere = await localChangeHeldOnlyHere(appId); + if (heldOnlyHere === null) { + log.warn(`revertLocalChangesIfSynced - ${appId}: could not read which of its ${changed} local change(s) other devices hold; neither reverting nor promoting this pass`); + return true; + } + if (heldOnlyHere) { + log.warn(`revertLocalChangesIfSynced - ${appId} holds local changes no other device holds (${files} files, ${directories} directories); keeping them, a revert would delete them`); + fluxEventBus.publish('syncthing:localChangesKept', { folder: appId, files, directories }); + return false; + } + log.warn(`revertLocalChangesIfSynced - ${appId} is synced but the receive-only folder has ${changed} locally changed item(s) (${files} files, ${directories} directories); reverting to the cluster's version`); try { // dataOrThrow: dbRevert answers in-band; without it this catch is // dead code and a failed revert reads as reverted messageHelper.dataOrThrow(await syncthingService.dbRevert(appId)); - fluxEventBus.publish('syncthing:localChangesReverted', { - folder: appId, - files: syncStatus.receiveOnlyChangedFiles || 0, - directories: syncStatus.receiveOnlyChangedDirectories || 0, - }); + fluxEventBus.publish('syncthing:localChangesReverted', { folder: appId, files, directories }); } catch (error) { log.error(`revertLocalChangesIfSynced - revert of local changes for ${appId} failed: ${error.message}`); } @@ -1708,8 +1754,9 @@ async function handleReceiveOnlyTransition(params) { // data: local additions in a receiveonly folder leave needBytes 0 / completion 100, // and promoting would broadcast those local changes cluster-wide. Verify the folder // is clean first; if not, revert the local changes (db/revert undoes local edits in - // a receiveonly folder) and promote on a later cycle once verifiably clean. The - // leader path above is exempt by design - the leader's local data IS the seed. + // a receiveonly folder) and promote on a later cycle once verifiably clean. Changes + // no other device holds are kept, and promoted with the rest of this node's data. + // The leader path above is exempt by design - the leader's local data IS the seed. if (await revertLocalChangesIfSynced(appId, syncStatus)) { return { syncthingFolder, cache }; } diff --git a/ZelBack/src/services/syncthingService.js b/ZelBack/src/services/syncthingService.js index 0780f52908..0e81be9fba 100644 --- a/ZelBack/src/services/syncthingService.js +++ b/ZelBack/src/services/syncthingService.js @@ -876,6 +876,17 @@ async function getDbLocalChanged(folder, page = 1, perpage = LOCAL_CHANGED_PAGE_ return request('get', `/rest/db/localchanged?folder=${folder}&page=${page}&perpage=${perpage}`); } +/** + * One item of a folder: this node's record of it (`local`), the cluster's version + * (`global`), and the connected devices that hold that version (`availability`). + * @param {string} folder Folder ID. + * @param {string} file The item's path within the folder. + * @returns {Promise<{local: object, global: object, availability: Array}>} + */ +async function getDbFile(folder, file) { + return request('get', `/rest/db/file?${qs.stringify({ folder, file })}`); +} + /** * A folder's local changes, handed to a caller one page at a time. * @@ -2737,6 +2748,7 @@ module.exports = { nonFileEntryBytes, nonFileEntryBytesForVersion, statusInFileBytes, + getDbFile, getDbLocalChanged, eachDbLocalChanged, dbRevert, diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 193e69bb92..3fb847a9fa 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -288,6 +288,7 @@ export function nodeClient(nodeNum) { 'network:originatorNotListed', 'syncthing:namesVisible', 'syncthing:localChangesReverted', + 'syncthing:localChangesKept', 'syncthing:devicesResumed', 'pm2:killTimeoutRaiseFailed', 'pm2:killTimeoutUnchanged', diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index db56a03a18..e350f27bf6 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -14,6 +14,8 @@ const syncthingService = require('../../ZelBack/src/services/syncthingService'); const syncthingServiceMock = { getDbStatus: sinon.stub(), eachDbLocalChanged: sinon.stub(), + getDbLocalChanged: sinon.stub(), + getDbFile: sinon.stub(), systemRestart: sinon.stub(), getConfig: sinon.stub(), getDbCompletion: sinon.stub(), @@ -214,6 +216,11 @@ describe('syncthingFolderStateMachine tests', () => { globalStateMock.syncthingDevicesIDCache.clear(); syncthingServiceMock.dbRevert.reset(); syncthingServiceMock.dbRevert.resolves({ status: 'success' }); + // default: one local change, whose cluster version another device holds + syncthingServiceMock.getDbLocalChanged.reset(); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'appdata/changed' }] }); + syncthingServiceMock.getDbFile.reset(); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 }, local: { localFlags: 8 } }); syncthingServiceMock.scanFolder.reset(); syncthingServiceMock.scanFolder.resolves({}); // default: this node's syncthing has never been connected to any peer @@ -2302,6 +2309,72 @@ describe('syncthingFolderStateMachine tests', () => { sinon.assert.neverCalledWith(appReconcilerMock.setControllerDesired, sinon.match.any, 'running'); }); + // A change whose cluster version is this node's own is held by no other device: + // syncthing reverts it by deleting it. It is this node's data, so it is kept and + // promoted with the rest. + describe('a local change no other device holds', () => { + const pollutedCandidate = () => { + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1 }); + mockParams.appLocation.resolves([ + { ip: '10.0.0.0:16127', runningSince: null, broadcastedAt: 1000 }, + { ip: '10.0.0.1:16127', runningSince: null, broadcastedAt: 1000 }, + ]); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, + }); + }; + + it('is kept, and the folder is promoted', async () => { + pollutedCandidate(); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { localFlags: 8 } }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted, 'a change only this node holds kept the folder from promoting').to.be.true; + }); + + it('is found behind changes other devices hold', async () => { + pollutedCandidate(); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'appdata/held' }, { name: 'appdata/only-here' }] }); + syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ global: { localFlags: 8 } }); + + await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('is kept on a ready standby', async () => { + mockParams.containerDataFlags = 'g'; + mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); + dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, + }); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 } }); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.calledWithExactly(publish, 'syncthing:localChangesKept', { folder: 'test-app', files: 0, directories: 1 }); + } finally { + publish.restore(); + } + }); + + it('is neither reverted nor promoted while which changes others hold cannot be read', async () => { + pollutedCandidate(); + syncthingServiceMock.getDbFile.rejects(new Error('connect ECONNREFUSED')); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + }); + }); + // Three properties of what counts as "holding", each able to hand the seed to the // wrong node. In all three this node's address sorts FIRST, so the address order // would give it the seed; only the holdings decide otherwise. From c604a0e17fb6de824153712e42e6d85c3df41a02 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 16:49:11 +0100 Subject: [PATCH 270/369] fix(syncthing): only a change to data from another device is kept from a revert A change no other device holds is kept only when its history names another device: it is the only copy of data the app wrote elsewhere. One whose history is this node's alone was made here, and is reverted as any pollution is. Co-Authored-By: Claude Opus 5.5 --- .../syncthingFolderStateMachine.js | 31 +++++++++++++------ .../unit/syncthingFolderStateMachine.test.js | 21 ++++++++++--- 2 files changed, 37 insertions(+), 15 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 90715f3f44..ad62f61021 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1293,15 +1293,23 @@ const RECEIVE_ONLY_CHANGE_FLAG = 8; const LOCAL_CHANGE_PAGE_SIZE = 1000; /** - * Whether any of a receive-only folder's local changes is held by no other device: - * the cluster's version of it is this node's own change. syncthing reverts such an - * item by deleting it, and a directory that is not empty cannot be deleted, so that - * revert fails on every pass. + * Whether a receive-only folder holds a local change that is the cluster's data and + * that no other device holds: an item whose cluster version is this node's own change + * (db/file, global localFlags) and whose history names another device (a version + * vector counter that is not this node's). syncthing reverts such an item by deleting + * it - the only copy of data the app wrote elsewhere - and a directory that is not + * empty cannot be deleted, so that revert fails on every pass. + * + * An item whose history is this node's alone was made here, never by the app's + * writer: reverting deletes it, as it should. * @param {string} appId - The Syncthing folder ID * @returns {Promise} null when the changes could not be read */ async function localChangeHeldOnlyHere(appId) { try { + const deviceId = await syncthingService.getDeviceId(); + if (!deviceId) return null; + const ownCounter = `${deviceId.substring(0, 7)}:`; for (let page = 1; ; page += 1) { // eslint-disable-next-line no-await-in-loop const batch = (await syncthingService.getDbLocalChanged(appId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; @@ -1309,9 +1317,11 @@ async function localChangeHeldOnlyHere(appId) { // eslint-disable-next-line no-restricted-syntax for (const item of batch) { // eslint-disable-next-line no-await-in-loop - const { global } = await syncthingService.getDbFile(appId, item.name); + const { global, local } = await syncthingService.getDbFile(appId, item.name); // eslint-disable-next-line no-bitwise - if ((global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) return true; + const onlyHere = Boolean((global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG); + const fromElsewhere = (local?.version ?? []).some((counter) => !counter.startsWith(ownCounter)); + if (onlyHere && fromElsewhere) return true; } if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return false; } @@ -1330,9 +1340,10 @@ async function localChangeHeldOnlyHere(appId) { * version and deletes local-only items that are not ignored, so it runs only * against a complete global index (isSynced). * - * A change no other device holds is this node's data, and db/revert would delete - * it. A folder holding one is not reverted, and its changes do not hold the node - * back from taking over: promoted, it publishes them. + * A change no other device holds, to data that came from another device, is the + * only copy of it left, and db/revert would delete it. A folder holding one is not + * reverted - db/revert takes the whole folder, so its other changes stay too - and + * its changes do not hold the node back from taking over: promoted, it publishes them. * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status @@ -1351,7 +1362,7 @@ async function revertLocalChangesIfSynced(appId, syncStatus) { return true; } if (heldOnlyHere) { - log.warn(`revertLocalChangesIfSynced - ${appId} holds local changes no other device holds (${files} files, ${directories} directories); keeping them, a revert would delete them`); + log.warn(`revertLocalChangesIfSynced - ${appId} holds a local change to data from another device that no other device holds (${files} files, ${directories} directories changed); keeping its changes, a revert would delete it`); fluxEventBus.publish('syncthing:localChangesKept', { folder: appId, files, directories }); return false; } diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index e350f27bf6..5822d99a09 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2310,8 +2310,9 @@ describe('syncthingFolderStateMachine tests', () => { }); // A change whose cluster version is this node's own is held by no other device: - // syncthing reverts it by deleting it. It is this node's data, so it is kept and - // promoted with the rest. + // syncthing reverts it by deleting it. One to data another device wrote is the + // only copy of that data, so it is kept and promoted with the rest; one this node + // made itself is reverted, as any pollution is. describe('a local change no other device holds', () => { const pollutedCandidate = () => { mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1 }); @@ -2326,7 +2327,7 @@ describe('syncthingFolderStateMachine tests', () => { it('is kept, and the folder is promoted', async () => { pollutedCandidate(); - syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { localFlags: 8 } }); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { localFlags: 8, version: ['PRIMARY:5', 'LOCAL-D:9'] } }); const result = await stateMachine.manageFolderSyncState(mockParams); @@ -2334,10 +2335,20 @@ describe('syncthingFolderStateMachine tests', () => { expect(result.cache.restarted, 'a change only this node holds kept the folder from promoting').to.be.true; }); + it('is reverted when this node made it', async () => { + pollutedCandidate(); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { localFlags: 8, version: ['LOCAL-D:9'] } }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + expect(result.cache.restarted).to.be.false; + }); + it('is found behind changes other devices hold', async () => { pollutedCandidate(); syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'appdata/held' }, { name: 'appdata/only-here' }] }); - syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ global: { localFlags: 8 } }); + syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ global: { localFlags: 8 }, local: { version: ['PRIMARY:5'] } }); await stateMachine.manageFolderSyncState(mockParams); @@ -2352,7 +2363,7 @@ describe('syncthingFolderStateMachine tests', () => { syncthingServiceMock.getDbStatus.resolves({ globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, }); - syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 } }); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { version: ['PRIMARY:5', 'LOCAL-D:9'] } }); const publish = sinon.stub(fluxEventBus, 'publish'); try { await stateMachine.manageFolderSyncState(mockParams); From 561c2e2355df1db5939c08941cab17b0ce489bb3 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:02:31 +0100 Subject: [PATCH 271/369] test(harness): 3310, a primary that went quiet and then died is taken over A single-writer app's primary stops announcing while its syncthing and container carry on; its location expires on the standby, which keeps it in the folder and starts nothing. Then its machine dies; the standby's changes to the primary's data, now held by no other device, are kept, the standby takes over with the data intact, and a new standby syncs that copy. dbClient.expireAppLocation removes a node's location rows for an app, and placeSilentPrimaryApp returns the app it placed. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/db-client.js | 10 ++ test-infra/runner/framework/silent-primary.js | 1 + ...-went-quiet-and-then-died-is-taken-over.js | 128 ++++++++++++++++++ 3 files changed, 139 insertions(+) create mode 100644 test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index 36b18b9b33..aba0e5a523 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -149,6 +149,16 @@ export function dbClient(nodeNum) { return globalDb.collection('zelappslocation').find({ ip }).toArray(); }, + // A location's expiry on this node: its rows for the app at that address are + // gone, as the TTL index removes them. Resolves to how many rows went. + async expireAppLocation(appName, ip) { + const globalDb = await db('appsGlobal'); + const host = String(ip).replace(/:\d+$/, '').replace(/\./g, '\\.'); + const { deletedCount } = await globalDb.collection('zelappslocation') + .deleteMany({ name: appName, ip: { $regex: `^${host}(:\\d+)?$` } }); + return deletedCount; + }, + // The restart attempts this node's reconciler has recorded for a component: // the burst window and the backoff ladder, as many entries as each holds. async restartAttempts(identifier) { diff --git a/test-infra/runner/framework/silent-primary.js b/test-infra/runner/framework/silent-primary.js index ac9c0248a4..e50d453ce0 100644 --- a/test-infra/runner/framework/silent-primary.js +++ b/test-infra/runner/framework/silent-primary.js @@ -105,6 +105,7 @@ export async function placeSilentPrimaryApp({ return { env, + app, identifier, // When more than one node ran the component. twoRan, diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js new file mode 100644 index 0000000000..be0d545f2f --- /dev/null +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -0,0 +1,128 @@ +// weight: heavy +/* + * A single-writer app's primary stops announcing while it carries on running: its + * FluxOS is gone, its syncthing and its container are not. Its location expires, + * and the standby keeps it in the folder for as long as its syncthing is connected + * and shares the folder; nothing starts beside it. + * + * Then the machine dies. The standby drops it from the folder, and changes on the + * standby's disk to the data the primary wrote are now held by no other device: + * syncthing reverts such a change by deleting it. The standby keeps them, takes the + * app over with the primary's data intact, and a new standby syncs that copy. + * + * A = the standby, B = the primary, C = the new standby. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { crashFluxos, execInContainer, stopAppContainer } from '../framework/container.js'; +import { + getDeviceId, getConnectedDevices, getFolderConfig, getFileInfo, isDaemonUp, readPath, scanFolder, stopDaemon, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor } from '../framework/wait.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// The harness's syncthing monitor passes every 3s; this is several of them. +const MONITOR_PASSES_MS = 30000; +// A machine that dies closes none of its connections, so the standby's syncthing +// holds them until its own receive timeout (5 min, not compressed by the harness). +// Then a few election passes and a container start. +const TAKEOVER_MS = 10 * 60 * 1000; +// syncthing's FlagLocalReceiveOnly +const RECEIVE_ONLY_CHANGE_FLAG = 8; + +function step(what) { + console.log(`# 3310 ${new Date().toISOString()} ${what}`); +} + +describe('a primary that went quiet and then died is taken over', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2equietdied${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let primaryDevice; + let afterDeath; + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + primaryDevice = await getDeviceId(client(B)); + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('keeps a primary whose location expired in the folder while it is connected', async function () { + this.timeout(300000); + step('the primary stops announcing; its syncthing and app carry on'); + await crashFluxos(client(B).container, { hold: true }); + expect(await isDaemonUp(client(B)), 'the primary\'s syncthing went down with its FluxOS, so nothing here is tested').to.equal(true); + const expired = await Promise.all([A, C].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); + expect(expired[0], 'the standby held no location for the primary, so none expired').to.be.at.least(1); + + await sleepUnlessInfraDead(MONITOR_PASSES_MS); + expect(await getConnectedDevices(client(A)), 'the primary\'s syncthing is not connected to the standby').to.include(primaryDevice); + expect((await dbClient(A + 1).getAppLocations(appName)).length, 'a location for the primary came back').to.equal(0); + const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; + expect(devices, 'the standby dropped a connected primary from the folder').to.include(primaryDevice); + expect(fleet.twoRan, 'the standby started beside a primary that still runs').to.deep.equal([]); + }); + + it('takes over once that primary dies, keeping the primary\'s data only it holds', async function () { + this.timeout(TAKEOVER_MS + 300000); + step('the primary\'s machine dies'); + await stopAppContainer(client(B).container, appName, appName); + await stopDaemon(client(B)); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) + .some((d) => d.deviceID === primaryDevice), { + timeout: 600000, interval: 5000, label: 'the standby drops the dead primary from the folder', + }); + + step('the standby\'s copy of the primary\'s data changes on its disk'); + afterDeath = client(A).getLastEventId(); + const changed = await execInContainer(client(A).container, `chmod 600 ${root}/appdata/seed-data && chmod 700 ${root}/appdata`); + expect(changed.exitCode, `could not change the standby's copy: ${changed.output}`).to.equal(0); + await scanFolder(client(A), folder); + let record; + await waitFor(async () => { + record = await getFileInfo(client(A), folder, 'appdata/seed-data'); + // eslint-disable-next-line no-bitwise + return ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; + }, { timeout: 60000, interval: 2000, label: 'the change is one no other device holds' }); + expect(record.local.version.some((counter) => counter.startsWith(`${primaryDevice.substring(0, 7)}:`)), + 'the changed file carries no history from the primary, so it is not the primary\'s data').to.equal(true); + + step('waiting for the takeover'); + await waitFor(() => fleet.isUp(A), { + timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', + }); + expect((await readPath(client(A), `${root}/appdata/seed-data`))?.trim(), 'the primary\'s data is gone from the standby').to.equal('seeded'); + const kept = client(A).getEventBuffer().some((e) => e.id > afterDeath && e.event === 'syncthing:localChangesKept' + && e.data?.folder === folder); + expect(kept, 'the standby did not keep the change no other device holds').to.equal(true); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); + + it('gives a new standby the survivor\'s copy', async function () { + this.timeout(600000); + const afterInstall = client(C).getLastEventId(); + await installOnNodes(fleet.env, fleet.app, [C]); + await waitFor(async () => (await readPath(client(C), `${root}/appdata/seed-data`))?.trim() === 'seeded', { + timeout: 300000, interval: 5000, label: 'the new standby holds the primary\'s data', + }); + await client(C).waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 240000, { afterId: afterInstall }); + }); +}); From 12c48f791bb070dbdc27a56fe57771a1dd59777b Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:05:06 +0100 Subject: [PATCH 272/369] test(harness): 3310 counts only the primary's location rows as the expired ones Co-Authored-By: Claude Opus 5.5 --- ...0-a-primary-that-went-quiet-and-then-died-is-taken-over.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index be0d545f2f..1b58cb630f 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -75,7 +75,9 @@ describe('a primary that went quiet and then died is taken over', function () { await sleepUnlessInfraDead(MONITOR_PASSES_MS); expect(await getConnectedDevices(client(A)), 'the primary\'s syncthing is not connected to the standby').to.include(primaryDevice); - expect((await dbClient(A + 1).getAppLocations(appName)).length, 'a location for the primary came back').to.equal(0); + const host = (ip) => String(ip).split(':')[0]; + const primaryRows = (await dbClient(A + 1).getAppLocations(appName)).filter((row) => host(row.ip) === host(client(B).ip)); + expect(primaryRows.length, 'a location for the primary came back').to.equal(0); const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; expect(devices, 'the standby dropped a connected primary from the folder').to.include(primaryDevice); expect(fleet.twoRan, 'the standby started beside a primary that still runs').to.deep.equal([]); From 646e1f99295844c4eebe9b27269e280aaad76a21 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:18:05 +0100 Subject: [PATCH 273/369] test(harness): 3310 changes a file and a directory of the primary's, leaving the folder synced The primary writes a second file before it goes quiet. After it dies the standby's copy of that file and of its directory change, and the rest of the folder does not, so the copy still reads as synced and the revert is due: syncthing would delete the file and fail for ever on the directory. The standby keeps both and takes over with them. Co-Authored-By: Claude Opus 5.5 --- ...-went-quiet-and-then-died-is-taken-over.js | 33 ++++++++++++++----- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index 1b58cb630f..36bb40fd12 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -6,9 +6,12 @@ * and shares the folder; nothing starts beside it. * * Then the machine dies. The standby drops it from the folder, and changes on the - * standby's disk to the data the primary wrote are now held by no other device: - * syncthing reverts such a change by deleting it. The standby keeps them, takes the - * app over with the primary's data intact, and a new standby syncs that copy. + * standby's disk to a file and a directory the primary wrote are now held by no other + * device: syncthing reverts such a change by deleting it, which a directory that is + * not empty refuses for ever. The rest of the folder is unchanged, so the standby's + * copy still reads as synced and the revert is due. The standby keeps the changes, + * takes the app over with the primary's data intact, and a new standby syncs that + * copy. * * A = the standby, B = the primary, C = the new standby. */ @@ -38,6 +41,8 @@ const MONITOR_PASSES_MS = 30000; const TAKEOVER_MS = 10 * 60 * 1000; // syncthing's FlagLocalReceiveOnly const RECEIVE_ONLY_CHANGE_FLAG = 8; +const SAVE = 'appdata/save-b'; +const SAVE_CONTENT = 'written by the primary'; function step(what) { console.log(`# 3310 ${new Date().toISOString()} ${what}`); @@ -57,6 +62,12 @@ describe('a primary that went quiet and then died is taken over', function () { this.timeout(900000); fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); primaryDevice = await getDeviceId(client(B)); + const wrote = await execInContainer(client(B).container, `printf '${SAVE_CONTENT}' > ${root}/${SAVE}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on the primary: ${wrote.output}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 240000, interval: 3000, label: 'the primary\'s second file reached the standby', + }); }); after(async function () { @@ -95,22 +106,26 @@ describe('a primary that went quiet and then died is taken over', function () { step('the standby\'s copy of the primary\'s data changes on its disk'); afterDeath = client(A).getLastEventId(); - const changed = await execInContainer(client(A).container, `chmod 600 ${root}/appdata/seed-data && chmod 700 ${root}/appdata`); + const changed = await execInContainer(client(A).container, `chmod 600 ${root}/${SAVE} && chmod 700 ${root}/appdata`); expect(changed.exitCode, `could not change the standby's copy: ${changed.output}`).to.equal(0); await scanFolder(client(A), folder); + // eslint-disable-next-line no-bitwise + const heldOnlyHere = (record) => ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; let record; await waitFor(async () => { - record = await getFileInfo(client(A), folder, 'appdata/seed-data'); - // eslint-disable-next-line no-bitwise - return ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; - }, { timeout: 60000, interval: 2000, label: 'the change is one no other device holds' }); + record = await getFileInfo(client(A), folder, SAVE); + return heldOnlyHere(record) && heldOnlyHere(await getFileInfo(client(A), folder, 'appdata')); + }, { timeout: 60000, interval: 2000, label: 'the file and the directory are changes no other device holds' }); expect(record.local.version.some((counter) => counter.startsWith(`${primaryDevice.substring(0, 7)}:`)), 'the changed file carries no history from the primary, so it is not the primary\'s data').to.equal(true); + expect(heldOnlyHere(await getFileInfo(client(A), folder, 'appdata/seed-data')), + 'the unchanged file is held only here too, so the standby\'s copy no longer reads as synced').to.equal(false); step('waiting for the takeover'); await waitFor(() => fleet.isUp(A), { timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', }); + expect(await readPath(client(A), `${root}/${SAVE}`), 'the primary\'s file is gone from the standby').to.equal(SAVE_CONTENT); expect((await readPath(client(A), `${root}/appdata/seed-data`))?.trim(), 'the primary\'s data is gone from the standby').to.equal('seeded'); const kept = client(A).getEventBuffer().some((e) => e.id > afterDeath && e.event === 'syncthing:localChangesKept' && e.data?.folder === folder); @@ -122,7 +137,7 @@ describe('a primary that went quiet and then died is taken over', function () { this.timeout(600000); const afterInstall = client(C).getLastEventId(); await installOnNodes(fleet.env, fleet.app, [C]); - await waitFor(async () => (await readPath(client(C), `${root}/appdata/seed-data`))?.trim() === 'seeded', { + await waitFor(async () => (await readPath(client(C), `${root}/${SAVE}`)) === SAVE_CONTENT, { timeout: 300000, interval: 5000, label: 'the new standby holds the primary\'s data', }); await client(C).waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 240000, { afterId: afterInstall }); From 8eebabc9ce5d490ec8793c3273579f5ba61fa655 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:18:06 +0100 Subject: [PATCH 274/369] fix(syncthing): a device seen within a day keeps its entry when no folder uses it Its connection record is what proves a peer gone. A standby that remembers a primary no location names any more still has to prove that primary gone before it starts the app, and with the device's entry deleted it never could, so the app stayed down. The device sweep keeps an unused device that syncthing saw within UNUSED_DEVICE_KEEP_MS (24 h), and removes no device while when each was last seen cannot be read. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/syncthingMonitor.js | 15 +++++++++-- tests/unit/syncthingMonitor.test.js | 25 +++++++++++++++++++ 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 1dc176a551..b88dc1ecba 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -52,6 +52,12 @@ const syncthingEventsConsumer = require('./syncthingEventsConsumer'); // Global collections const globalAppsLocations = config.database.appsglobal.collections.appsLocations; +// How long a device no folder uses keeps its entry after syncthing last saw it. +// Its connection record is what proves a peer gone (peerFolderLiveness), and a +// standby that remembers a primary no location names any more still has to prove +// that primary gone before it starts the app; with the entry deleted it never can. +const UNUSED_DEVICE_KEEP_MS = 24 * 60 * 60 * 1000; + // Path constants const fluxDirPath = process.env.FLUXOS_PATH || path.join(process.env.HOME, 'zelflux'); const appsFolderPath = process.env.FLUX_APPS_FOLDER || path.join(fluxDirPath, 'ZelApps'); @@ -930,8 +936,13 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // A peer device cannot be attributed to an app without doing that app's // work, so while anything is held out the device sweep stands down entirely // rather than delete a peer it simply never saw. - const nonUsedDevices = unsafeFolderIds.size > 0 ? [] : allDevices.filter( - (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId, + // A device seen within UNUSED_DEVICE_KEEP_MS stays too, and every device stays + // while when each was last seen cannot be read. + const deviceStats = unsafeFolderIds.size > 0 ? null : await syncthingService.getDeviceStats().catch(() => null); + const seenRecently = (deviceID) => Date.now() - Date.parse(deviceStats?.[deviceID]?.lastSeen) < UNUSED_DEVICE_KEEP_MS; + const nonUsedDevices = !deviceStats ? [] : allDevices.filter( + (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId + && !seenRecently(syncthingDevice.deviceID), ); // Parallelize cleanup operations diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index 232eecef5a..25ed034ded 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -37,6 +37,7 @@ const syncthingServiceMock = { systemResume: sinon.stub().resolves(), getDbStatus: sinon.stub(), getDbCompletion: sinon.stub().resolves({ remoteState: 'notSharing' }), + getDeviceStats: sinon.stub().resolves({}), }; const syncthingFolderStateMachineMock = { @@ -1868,6 +1869,8 @@ describe('syncthingMonitor tests', () => { afterEach(() => { syncthingServiceMock.getDbCompletion.reset(); syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'notSharing' }); + syncthingServiceMock.getDeviceStats.reset(); + syncthingServiceMock.getDeviceStats.resolves({}); }); it('keeps its device while it is connected and shares the folder', async () => { @@ -1888,6 +1891,28 @@ describe('syncthingMonitor tests', () => { sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); }); + + // Its connection record is what proves it gone to a standby that remembers it as + // the app's primary. + it('keeps the entry of a device it dropped while syncthing saw it within a day', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getDeviceStats.resolves({ 'PEER-DEVICE': { lastSeen: new Date(Date.now() - 60 * 60 * 1000).toISOString() } }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + + it('removes no device while when each was last seen cannot be read', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getDeviceStats.rejects(new Error('connect ECONNREFUSED')); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); }); describe('unused-folder cleanup', () => { From b0ce6fc43efa63a44b80d76a0749b8acf0506bd6 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:36:13 +0100 Subject: [PATCH 275/369] fix(syncthing): a folder keeping its local changes says so once, when it starts The folder's cache entry carries keepingChanges, so the warning and syncthing:localChangesKept come once per stretch of kept changes rather than on every monitor pass. Co-Authored-By: Claude Opus 5.5 --- .../syncthingFolderStateMachine.js | 17 +++++++++++++---- tests/unit/syncthingFolderStateMachine.test.js | 7 ++++++- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index ad62f61021..cc1f166a09 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1347,13 +1347,19 @@ async function localChangeHeldOnlyHere(appId) { * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status + * @param {Object} [folderCache] - The folder's cache entry: `keepingChanges` marks a folder + * already said to be keeping its changes, so that is said once, when it starts * @returns {Promise} True when the folder has local changes it does not * keep: a revert was asked for, or which changes other devices hold could not be read */ -async function revertLocalChangesIfSynced(appId, syncStatus) { +async function revertLocalChangesIfSynced(appId, syncStatus, folderCache) { + const cache = folderCache || {}; if (!syncStatus?.isSynced) return false; const changed = (syncStatus.receiveOnlyChangedFiles || 0) + (syncStatus.receiveOnlyChangedDirectories || 0); - if (changed === 0) return false; + if (changed === 0) { + delete cache.keepingChanges; + return false; + } const files = syncStatus.receiveOnlyChangedFiles || 0; const directories = syncStatus.receiveOnlyChangedDirectories || 0; const heldOnlyHere = await localChangeHeldOnlyHere(appId); @@ -1362,10 +1368,13 @@ async function revertLocalChangesIfSynced(appId, syncStatus) { return true; } if (heldOnlyHere) { + if (cache.keepingChanges) return false; + cache.keepingChanges = true; log.warn(`revertLocalChangesIfSynced - ${appId} holds a local change to data from another device that no other device holds (${files} files, ${directories} directories changed); keeping its changes, a revert would delete it`); fluxEventBus.publish('syncthing:localChangesKept', { folder: appId, files, directories }); return false; } + delete cache.keepingChanges; log.warn(`revertLocalChangesIfSynced - ${appId} is synced but the receive-only folder has ${changed} locally changed item(s) (${files} files, ${directories} directories); reverting to the cluster's version`); try { // dataOrThrow: dbRevert answers in-band; without it this catch is @@ -1768,7 +1777,7 @@ async function handleReceiveOnlyTransition(params) { // a receiveonly folder) and promote on a later cycle once verifiably clean. Changes // no other device holds are kept, and promoted with the rest of this node's data. // The leader path above is exempt by design - the leader's local data IS the seed. - if (await revertLocalChangesIfSynced(appId, syncStatus)) { + if (await revertLocalChangesIfSynced(appId, syncStatus, cache)) { return { syncthingFolder, cache }; } if (syncStatus.isSynced) { @@ -2083,7 +2092,7 @@ async function manageFolderSyncState(params) { // deviation (an owner or mode included) is never overwritten by syncthing // in a receiveonly folder, and the primary counts it as outstanding. if (syncFolder?.type === 'receiveonly') { - await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId)); + await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId), cache); } return { syncthingFolder, cache }; } diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 5822d99a09..045dd2250b 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2367,9 +2367,14 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { await stateMachine.manageFolderSyncState(mockParams); + await stateMachine.manageFolderSyncState(mockParams); sinon.assert.notCalled(syncthingServiceMock.dbRevert); - sinon.assert.calledWithExactly(publish, 'syncthing:localChangesKept', { folder: 'test-app', files: 0, directories: 1 }); + sinon.assert.calledOnceWithExactly( + publish.withArgs('syncthing:localChangesKept'), + 'syncthing:localChangesKept', + { folder: 'test-app', files: 0, directories: 1 }, + ); } finally { publish.restore(); } From ef17c7d8bd59a21cffd5a1f751dc944cf373b3f5 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:36:14 +0100 Subject: [PATCH 276/369] test(harness): 3310 clears FDM's primary once the primary dies FDM drops a primary whose app stops answering, as suite 2228 has it; left named, the standby rightly defers to it. Co-Authored-By: Claude Opus 5.5 --- ...10-a-primary-that-went-quiet-and-then-died-is-taken-over.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index 36bb40fd12..28971b544b 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -28,6 +28,7 @@ import { } from '../framework/syncthing-real.js'; import { dbClient } from '../framework/db-client.js'; import { installOnNodes } from '../framework/reconciler-suite.js'; +import { clearMaster } from '../framework/fdm-control.js'; import { waitFor } from '../framework/wait.js'; import { sleepUnlessInfraDead } from '../framework/infra-death.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -99,6 +100,8 @@ describe('a primary that went quiet and then died is taken over', function () { step('the primary\'s machine dies'); await stopAppContainer(client(B).container, appName, appName); await stopDaemon(client(B)); + // FDM drops a primary whose app stops answering + await clearMaster(appName); await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) .some((d) => d.deviceID === primaryDevice), { timeout: 600000, interval: 5000, label: 'the standby drops the dead primary from the folder', From b8719552e81e0bd974ebd77ab76c4439af8efb65 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:40:07 +0100 Subject: [PATCH 277/369] fix(syncthing): a device kept in a folder no location names is said once, when it starts Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMonitoring/syncthingMonitor.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index b88dc1ecba..f134780129 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -255,6 +255,10 @@ async function readAppLocation(appName) { return results || []; } +// `|` for each device connectedUnnamedDevices keeps, so keeping +// it is said once. +const keptUnnamedDevices = new Set(); + /** * The devices of a folder that its app's locations no longer name but that are * still connected and sharing it. @@ -390,8 +394,13 @@ async function processContainerData(params) { if (syncFolder) { const kept = await connectedUnnamedDevices(id, syncFolder.devices, devices); + [...keptUnnamedDevices].filter((key) => key.startsWith(`${id}|`) && !kept.includes(key.slice(id.length + 1))) + .forEach((key) => keptUnnamedDevices.delete(key)); kept.forEach((deviceID) => { - log.info(`processContainerData - ${appId}: keeping device ${deviceID.substring(0, 7)}, connected and sharing the folder though no location names it`); + if (!keptUnnamedDevices.has(`${id}|${deviceID}`)) { + keptUnnamedDevices.add(`${id}|${deviceID}`); + log.info(`processContainerData - ${appId}: keeping device ${deviceID.substring(0, 7)}, connected and sharing the folder though no location names it`); + } devices.push({ deviceID }); if (!devicesIds.includes(deviceID)) devicesIds.push(deviceID); }); From b4a9ad7ecb0bb279f707167964d024ab7e692106 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 17:40:07 +0100 Subject: [PATCH 278/369] test(harness): 3310 keeps the dead primary named on FDM while the standby judges its changes FDM is cleared only after several monitor passes, so the standby is still a standby when it decides whether to revert the changes no other device holds; cleared at once, the takeover won the race and the decision was never made. Co-Authored-By: Claude Opus 5.5 --- ...-primary-that-went-quiet-and-then-died-is-taken-over.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index 28971b544b..ecd92d9d18 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -100,8 +100,6 @@ describe('a primary that went quiet and then died is taken over', function () { step('the primary\'s machine dies'); await stopAppContainer(client(B).container, appName, appName); await stopDaemon(client(B)); - // FDM drops a primary whose app stops answering - await clearMaster(appName); await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) .some((d) => d.deviceID === primaryDevice), { timeout: 600000, interval: 5000, label: 'the standby drops the dead primary from the folder', @@ -124,6 +122,11 @@ describe('a primary that went quiet and then died is taken over', function () { expect(heldOnlyHere(await getFileInfo(client(A), folder, 'appdata/seed-data')), 'the unchanged file is held only here too, so the standby\'s copy no longer reads as synced').to.equal(false); + // FDM still names the dead primary, so the standby stays a standby while its + // passes judge the change; then FDM drops it, as it drops a primary whose app + // stops answering. + await sleepUnlessInfraDead(MONITOR_PASSES_MS); + await clearMaster(appName); step('waiting for the takeover'); await waitFor(() => fleet.isUp(A), { timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', From 41ea0f0ab08ee54ca4ac34bc00de9ab0077b4c28 Mon Sep 17 00:00:00 2001 From: David White Date: Mon, 19 Oct 2026 19:40:08 +0100 Subject: [PATCH 279/369] fix(syncthing): a recently seen unused device under another device's name is not kept An unused device seen within a day keeps its entry, so a standby can still show a dead primary gone; one whose name another configured device also carries is not the device at that address, and the sweep removes it as before however recently it was seen. Co-Authored-By: Claude Opus 5.5 --- .../services/appMonitoring/syncthingMonitor.js | 9 ++++++--- tests/unit/syncthingMonitor.test.js | 15 +++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index f134780129..793f986b19 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -945,13 +945,16 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // A peer device cannot be attributed to an app without doing that app's // work, so while anything is held out the device sweep stands down entirely // rather than delete a peer it simply never saw. - // A device seen within UNUSED_DEVICE_KEEP_MS stays too, and every device stays - // while when each was last seen cannot be read. + // A device seen within UNUSED_DEVICE_KEEP_MS stays too, unless another device + // is configured under its name - then it is not the device at that address - + // and every device stays while when each was last seen cannot be read. const deviceStats = unsafeFolderIds.size > 0 ? null : await syncthingService.getDeviceStats().catch(() => null); const seenRecently = (deviceID) => Date.now() - Date.parse(deviceStats?.[deviceID]?.lastSeen) < UNUSED_DEVICE_KEEP_MS; + const nameHeldByAnother = (device) => Boolean(device.name) + && allDevices.some((other) => other.deviceID !== device.deviceID && other.name === device.name); const nonUsedDevices = !deviceStats ? [] : allDevices.filter( (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId - && !seenRecently(syncthingDevice.deviceID), + && (!seenRecently(syncthingDevice.deviceID) || nameHeldByAnother(syncthingDevice)), ); // Parallelize cleanup operations diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index 25ed034ded..f725038368 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -1904,6 +1904,21 @@ describe('syncthingMonitor tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); }); + it('removes a device seen within a day when another device holds its name', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getConfigDevices.resolves([ + { deviceID: 'PEER-DEVICE', name: '10.0.0.2:16127' }, + { deviceID: 'STALE-DEVICE', name: '10.0.0.2:16127' }, + ]); + const recent = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + syncthingServiceMock.getDeviceStats.resolves({ 'PEER-DEVICE': { lastSeen: recent }, 'STALE-DEVICE': { lastSeen: recent } }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'STALE-DEVICE'); + }); + it('removes no device while when each was last seen cannot be read', async () => { folderSharedWithPeer(); syncthingServiceMock.getDeviceStats.rejects(new Error('connect ECONNREFUSED')); From ae4b6e8d4d010dbddc9c22b5746c226774d60c19 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:19:56 +0100 Subject: [PATCH 280/369] fix(syncthing): nobody seeds a folder while one of its holders has not yet measured what it holds A holder answers what it holds of a folder once its monitor has measured it. Between its install and that pass it has measured nothing, and read then it holds nothing, so the field fell to the address order over the node holding the owner's data. The signed promoted-folders answer now lists, as notLookedAt, every synced folder of an installed app the node has neither measured nor made writable, and a node deciding a cold-start seed waits while any holder lists the folder or one of the app's other elected folders. A folder whose holdings cannot be read is recorded as unreadable rather than removed: no claim, as before, and not a folder still to be looked at. A node that cannot read its installed apps answers not ready. A peer that does not send the list reads as having none, so a mixed fleet decides as it did; an older node verifies the answer whole and reads the unreadable record as no claim. The installed apps are read through installedAppSpecs, which rejects when they cannot be read; the installed apps route answers through it as before. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/peerFolderLiveness.js | 21 +++--- .../syncthingFolderStateMachine.js | 36 ++++++--- .../src/services/appQuery/appQueryService.js | 62 ++++++++++++---- ZelBack/src/services/utils/fluxEventBus.js | 3 + ZelBack/src/services/utils/mountParser.js | 15 ++++ tests/unit/appQueryService.test.js | 60 ++++++++++++++- tests/unit/mountParser.test.js | 25 +++++++ tests/unit/peerFolderLiveness.test.js | 14 ++-- .../unit/syncthingFolderStateMachine.test.js | 73 ++++++++++++++++++- 9 files changed, 263 insertions(+), 46 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js index 5736111607..48fb81a8fa 100644 --- a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js +++ b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js @@ -76,7 +76,7 @@ async function holdingsRequest(socketAddr, challenge) { /** * A peer's answer as the callers read it. * @param {object} answer The `data` of the reply. - * @returns {{reachable: boolean, answerable: boolean, ready: boolean, folders: string[], holding: object, seeding: object}} + * @returns {{reachable: boolean, answerable: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[]}} */ function readAnswer(answer) { // A peer that has not completed its first monitor pass cannot tell "I hold @@ -90,8 +90,11 @@ function readAnswer(answer) { // Where the peer stands in deciding a cold-start seed, per folder. Absent reads as // nowhere, for the same reasons. const seeding = (answer && typeof answer.seeding === 'object' && answer.seeding) || {}; + // The folders the peer holds and has not yet measured. Absent on a peer too old to + // say, which reads as none, as that node's answer always has. + const notLookedAt = Array.isArray(answer?.notLookedAt) ? answer.notLookedAt : []; return { - reachable: true, answerable: true, ready, folders, holding, seeding, + reachable: true, answerable: true, ready, folders, holding, seeding, notLookedAt, }; } @@ -101,7 +104,7 @@ function readAnswer(answer) { // may be the one holding the copy. function notReady() { return { - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }; } @@ -132,7 +135,7 @@ function notReady() { * somewhere else, and says nothing of the peer. * * @param {string} socketAddr Peer socket address - * @returns {Promise<{reachable: boolean, answerable: boolean, misrouted?: boolean, unproven?: boolean, ready: boolean, folders: string[], holding: object, seeding: object}>} + * @returns {Promise<{reachable: boolean, answerable: boolean, misrouted?: boolean, unproven?: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[]}>} */ async function probePeer(socketAddr) { const ip = extractIp(socketAddr); @@ -156,13 +159,13 @@ async function probePeer(socketAddr) { if (asked.verdict === IdentityVerdict.MISROUTED) { log.info(`peerFolderLiveness - a call to ${ip} was answered by ${asked.answeredAs}; nothing it holds is known`); return { - reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }; } if (asked.verdict === IdentityVerdict.UNREACHABLE) { log.info(`peerFolderLiveness - could not read ${ip}: ${asked.reason}`); return { - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }; } @@ -176,7 +179,7 @@ async function probePeer(socketAddr) { if (!asked.mayReadUnsigned) { log.info(`peerFolderLiveness - a reply from ${ip} does not prove it came from that node; nothing it holds is known`); return { - reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }; } @@ -196,12 +199,12 @@ async function probePeer(socketAddr) { } log.info(`peerFolderLiveness - ${ip} answered ${error.response.status} and cannot say which folders it holds`); return { - reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }; } log.info(`peerFolderLiveness - could not read ${ip}: ${error.message}`); return { - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }; } } diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index cc1f166a09..753b042ef5 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -649,6 +649,10 @@ function lowestIpHolder(allPeersList) { * @param {object} claim - a candidate's { bytes, newestModified }, or nothing * @returns {boolean} */ +// What a folder's holdings are recorded as when this node cannot read them: no claim, +// and not a folder it has yet to look at. +const UNREADABLE_HOLDINGS = Object.freeze({ unreadable: true }); + function isRankableClaim(claim) { return Number.isFinite(claim?.bytes) && Number.isFinite(claim.newestModified); } @@ -1457,6 +1461,7 @@ async function handleReceiveOnlyTransition(params) { } = params; const folderPath = syncthingFolder.path || `${appsFolder}${appId}/appdata`; + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_HOLDINGS, appId); // Whether any CONNECTED peer genuinely holds the data. Gates the election (a true // cold start - nobody serving - must still elect one seed instead of standing off) @@ -1484,15 +1489,12 @@ async function handleReceiveOnlyTransition(params) { // not a smaller claim, it is no claim - and standDownOnUnknown below refuses to lead // on exactly that. Left up, the last good figure says 5.8 GB to every peer while this // node refuses to act on it: they rank it first and defer, it defers to its own - // unknown, and nobody seeds for as long as the read keeps failing. Removed, a peer - // reads it as not having answered, which drops the field to the address order - the - // case that already exists for a peer too old to answer at all. - if (holdings) { - if (!globalState.folderHoldings) globalState.folderHoldings = new Map(); - globalState.folderHoldings.set(appId, holdings); - } else { - globalState.folderHoldings?.delete(appId); - } + // unknown, and nobody seeds for as long as the read keeps failing. Recorded as + // unreadable, a peer reads it as no claim, which drops the field to the address + // order - the case that already exists for a peer too old to answer at all - and the + // folder is not one this node has yet to look at (promotedFolderHoldings). + if (!globalState.folderHoldings) globalState.folderHoldings = new Map(); + globalState.folderHoldings.set(appId, holdings ?? UNREADABLE_HOLDINGS); // Designated-leader election, debounced: require leadership to hold for // LEADER_CONFIRM_COUNT consecutive cycles, so a single transient peer-visibility blip // doesn't flip a follower to leader. @@ -1531,12 +1533,21 @@ async function handleReceiveOnlyTransition(params) { // its address differ, and the ranking degrades to the address order silently. const ownClaim = electedAppClaim(holdings, electedSiblings.map((sibling) => globalState.folderHoldings?.get(sibling))); const claims = {}; + // Candidates that hold this app and have not yet measured what they hold of it - an + // install their monitor has not reached. Ranked now, one would read as holding + // nothing and the field would fall to the address order over a holder that may hold + // the owner's data, so nobody seeds this pass. + const notLookedAt = []; await Promise.all(electionList.map(async (peer) => { if (socketAddressesMatch(peer.ip, localSocketAddr)) { if (ownClaim) claims[peer.ip] = ownClaim; return; } const answer = await liveness.read(peer.ip); + if (answer?.ready && [appId, ...electedSiblings].some((folder) => (answer.notLookedAt ?? []).includes(folder))) { + notLookedAt.push(peer.ip); + return; + } // Only a READY peer's claim counts - before its first monitor pass a node cannot // tell "I hold nothing" from "I have not looked". Defence in depth rather than // the thing that enforces it: a peer that has not determined its folder state @@ -1562,9 +1573,14 @@ async function handleReceiveOnlyTransition(params) { const appWriter = electedSiblings.length ? await electedAppWriter(electedSiblings, electionList, localSocketAddr, liveness) : null; + if (notLookedAt.length) { + fluxEventBus.count('syncthing:seedDecision', appId, 'holderNotLookedAt'); + if (!cache.waitingOnLook) log.info(`handleReceiveOnlyTransition - ${appId}: ${notLookedAt.join(', ')} has not yet measured what it holds; nobody seeds until it has`); + } + cache.waitingOnLook = notLookedAt.length > 0; const electedLeader = appWriter ? appWriter.here && !aPeerHasData - : !standDownOnUnknown && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); + : !standDownOnUnknown && !notLookedAt.length && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); // The floor holderIsGone asks of a silent holder, asked of this node before // its own win can count: a node whose peers have gone quiet is the one that // fell over, and a win it confirms in that state seeds the app on a diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index 7fa95b26d8..a77c7faadd 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -138,6 +138,17 @@ async function decryptEnterpriseApps(apps, options = {}) { return { readable, unreadable, inPlace }; } +/** + * The specifications of the apps installed on this node, from its own table. + * @param {object} [query] Narrows the read, as a filter on the table. + * @returns {Promise} + * @throws {Error} When the table cannot be read. + */ +async function installedAppSpecs(query = {}) { + const appsDatabase = dbHelper.databaseConnection().db(config.database.appslocal.database); + return dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, query, { projection: { _id: 0 } }); +} + /** * The names of the apps installed on this node that the network lists at * `socketAddress`: what a node that held that address left there. Rejects @@ -146,12 +157,11 @@ async function decryptEnterpriseApps(apps, options = {}) { * @returns {Promise} */ async function installedAppsListedAt(socketAddress) { - const installed = await installedApps(); - if (installed.status !== 'success') throw new Error(`installed apps unreadable: ${installed.data?.message ?? installed.data}`); - const names = new Set(installed.data.map((app) => app.name)); + const names = new Set((await installedAppSpecs()).map((app) => app.name)); return (await registryManager.appsListedAt(socketAddress)).filter((name) => names.has(name)); } + /** * To list installed apps. Returns apps from local database. * @param {object} req Request. @@ -160,9 +170,6 @@ async function installedAppsListedAt(socketAddress) { */ async function installedApps(req, res) { try { - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - let appsQuery = {}; if (req && req.params && req.query) { let { appname } = req.params; @@ -174,11 +181,7 @@ async function installedApps(req, res) { appsQuery = { name: req }; } - const appsProjection = { - projection: { _id: 0 }, - }; - - const apps = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); + const apps = await installedAppSpecs(appsQuery); const dataResponse = messageHelper.createDataMessage(apps); return res ? res.json(dataResponse) : dataResponse; } catch (error) { @@ -826,8 +829,29 @@ async function callerIsFluxnode(body) { * cold start (globalState.seedMarks); it carries the same figures as `holding`, so it * goes only where `holding` does. * - * @returns {object} Message carrying { ready, folders, holding, seeding }. + * `notLookedAt` is every synced folder of an installed app this node has neither + * measured nor made writable: an install its monitor has not yet reached. A peer about + * to seed waits on them, so a holder that has not looked is never read as holding + * nothing. A node that cannot read its installed apps cannot say, and answers not ready. + * + * @returns {object} Message carrying { ready, folders, notLookedAt, holding, seeding }. */ +/** + * The synced folders of the apps installed here that this node has neither measured + * nor made writable. + * @param {Set} writable The folders this node holds writable. + * @param {Map|null} held The folders it has measured. + * @returns {Promise} + * @throws {Error} When the installed apps cannot be read. + */ +async function foldersNotLookedAt(writable, held) { + const { readable } = await decryptEnterpriseApps(await installedAppSpecs()); + return readable + .flatMap((app) => mountParser.syncedComponentIdentifiers(app)) + .map((identifier) => dockerService.getAppIdentifier(identifier)) + .filter((folder) => !writable.has(folder) && !held?.has(folder)); +} + async function promotedFolderHoldings(req, res) { try { // eslint-disable-next-line global-require @@ -857,10 +881,21 @@ async function promotedFolderHoldings(req, res) { const entitled = authorized === true || await callerIsFluxnode(body); const held = globalState.folderHoldings; + let ready = ids !== null; + let notLookedAt = []; + if (ready) { + try { + notLookedAt = await foldersNotLookedAt(ids, held); + } catch (error) { + log.warn(`promotedFolderHoldings - the installed apps could not be read, so which folders are measured cannot be said; answering not ready: ${error.message}`); + ready = false; + } + } const seal = peerIdentityService.answerSealer(peerIdentityService.AnswerPurpose.PROMOTED_FOLDERS, body); const response = messageHelper.createDataMessage(await seal({ - ready: ids !== null, + ready, folders: ids === null ? [] : [...ids], + notLookedAt, ...(entitled ? { holding: held === null ? {} : Object.fromEntries(held), seeding: Object.fromEntries(globalState.seedMarks), @@ -879,6 +914,7 @@ async function promotedFolderHoldings(req, res) { } module.exports = { + installedAppSpecs, installedApps, installedAppsListedAt, decryptEnterpriseApps, diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index fe2ec33917..46eca167ad 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -68,6 +68,9 @@ const Checkpoint = Object.freeze({ // The elected leader has read its sync status and has not yet checked the disk // it would seed from holds what that status claims. Keyed by folder id. SYNCTHING_BEFORE_SEED_CHECK: 'syncthing:beforeSeedCheck', + // The syncthing monitor has reached a receive-only folder and not yet measured + // what this node holds in it. Keyed by folder id. + SYNCTHING_BEFORE_HOLDINGS: 'syncthing:beforeHoldings', // The syncthing monitor has found a single-writer folder sending on a node // that does not hold it, and has not yet asked for it to be held as a // standby. Keyed by folder id. diff --git a/ZelBack/src/services/utils/mountParser.js b/ZelBack/src/services/utils/mountParser.js index 215e1062a9..d9de884a29 100644 --- a/ZelBack/src/services/utils/mountParser.js +++ b/ZelBack/src/services/utils/mountParser.js @@ -545,6 +545,20 @@ function gComponentIdentifiers(appSpec) { .map((comp) => `${comp.name}_${appSpec.name}`); } +/** + * The components of an app whose data is synced (g:/r:/s:), in compose order, as + * `_` (the app name for a v1-3 app). Empty when it has none. + * @param {object} appSpec - Installed app specification, decrypted. + * @returns {string[]} + */ +function syncedComponentIdentifiers(appSpec) { + if (!appSpec) return []; + if (appSpec.version <= 3) return isSyncedComponent(appSpec.containerData) ? [appSpec.name] : []; + return (appSpec.compose || []) + .filter((comp) => isSyncedComponent(comp.containerData)) + .map((comp) => `${comp.name}_${appSpec.name}`); +} + /** * An app's anchor: its first g: component, or null when it has none. * @@ -591,6 +605,7 @@ module.exports = { getComponentSyncMode, isGComponent, gComponentIdentifiers, + syncedComponentIdentifiers, anchorComponentIdentifier, isSyncedComponent, validateMountPath, diff --git a/tests/unit/appQueryService.test.js b/tests/unit/appQueryService.test.js index 67ccd60f01..14265bb8ec 100644 --- a/tests/unit/appQueryService.test.js +++ b/tests/unit/appQueryService.test.js @@ -1027,6 +1027,9 @@ describe('appQueryService tests', () => { globalState.promotedFolderIds = new Set(['fluxa_a']); globalState.folderHoldings = new Map([['fluxa_a', { bytes: 5821604997, newestModified: 200 }]]); messageHelperStub.createDataMessage.returnsArg(0); + // no app installed unless a test says otherwise + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('appsDatabase') }); + dbHelperStub.findInDatabase.resolves([]); }); afterEach(() => { @@ -1071,7 +1074,7 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); expect(result.holding, 'holdings went to a caller that proved nothing').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'] }); + expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'], notLookedAt: [] }); }); }); @@ -1088,7 +1091,7 @@ describe('appQueryService tests', () => { }); expect(result.holding, 'a body dated in the future served as a credential').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'] }); + expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'], notLookedAt: [] }); }); // The bound is the fleet's clocks disagreeing, which is honest and must still work. @@ -1147,7 +1150,7 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body: signed({ target: '10.0.0.8:16127' }) }); expect(result.holding, 'a signature captured in flight worked on another node').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'] }); + expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'], notLookedAt: [] }); }); it('withholds the holdings when this node does not know its own address', async () => { @@ -1183,10 +1186,59 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); sinon.assert.calledOnceWithExactly(peerIdentityServiceStub.answerSealer, 'promoted', body); - expect(seal.firstCall.args[0]).to.have.keys('ready', 'folders', 'holding', 'seeding'); + expect(seal.firstCall.args[0]).to.have.keys('ready', 'folders', 'notLookedAt', 'holding', 'seeding'); expect(result.sealed).to.equal(true); }); + // An install its monitor has not reached: a peer about to seed waits on it rather + // than read this node as holding nothing. + describe('the folders it holds and has not yet measured', () => { + const synced = (name, mode) => ({ name, version: 8, compose: [{ name, containerData: `${mode}:/data` }] }); + + it('lists a synced folder of an installed app that it has neither measured nor made writable, to any caller', async () => { + dbHelperStub.findInDatabase.resolves([synced('b', 'r'), synced('c', 'g')]); + + const result = await appQueryService.promotedFolderHoldings({ body: {} }); + + expect(result.ready).to.equal(true); + expect(result.notLookedAt).to.have.members(['fluxb_b', 'fluxc_c']); + }); + + it('leaves out a folder it has measured, one it could not read, one it holds writable, and one that does not sync', async () => { + globalState.promotedFolderIds = new Set(['fluxa_a', 'fluxw_w']); + globalState.folderHoldings = new Map([ + ['fluxm_m', { bytes: 10, newestModified: 1 }], + ['fluxu_u', { unreadable: true }], + ]); + dbHelperStub.findInDatabase.resolves([ + synced('m', 'r'), synced('u', 'r'), synced('w', 's'), + { name: 'p', version: 8, compose: [{ name: 'p', containerData: '/data' }] }, + ]); + + const result = await appQueryService.promotedFolderHoldings({ body: {} }); + + expect(result.notLookedAt).to.deep.equal([]); + }); + + it('lists every synced folder before its monitor has measured any', async () => { + globalState.folderHoldings = null; + dbHelperStub.findInDatabase.resolves([synced('b', 'r')]); + + const result = await appQueryService.promotedFolderHoldings({ body: {} }); + + expect(result.notLookedAt).to.deep.equal(['fluxb_b']); + }); + + it('answers not ready when it cannot read its installed apps', async () => { + dbHelperStub.findInDatabase.rejects(new Error('operation exceeded time limit')); + + const result = await appQueryService.promotedFolderHoldings({ body: {} }); + + expect(result.ready, 'a node that cannot say which folders it has measured answered ready').to.equal(false); + expect(result.notLookedAt).to.deep.equal([]); + }); + }); + it('answers an error when the answer cannot be signed, rather than send it unsigned', async () => { messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); peerIdentityServiceStub.answerSealer.returns(() => Promise.reject(new Error('This node cannot sign as itself'))); diff --git a/tests/unit/mountParser.test.js b/tests/unit/mountParser.test.js index 311031ef4b..682bf9b4b7 100644 --- a/tests/unit/mountParser.test.js +++ b/tests/unit/mountParser.test.js @@ -443,6 +443,31 @@ describe('mountParser tests', () => { // The election decides the app and runs every g: component on its primary; an // operator start defers to it for the same set. Both read it here, so they // cannot disagree. + describe('syncedComponentIdentifiers', () => { + it('names every synced component of a composed app, in compose order', () => { + const spec = { + name: 'App', + version: 4, + compose: [ + { name: 'two', containerData: 'g:/data' }, + { name: 'db', containerData: '/data' }, + { name: 'cache', containerData: 'r:/data' }, + { name: 'shared', containerData: 's:/data' }, + ], + }; + expect(mountParser.syncedComponentIdentifiers(spec)).to.deep.equal(['two_App', 'cache_App', 'shared_App']); + }); + + it('names a v1-3 synced app by its own name', () => { + expect(mountParser.syncedComponentIdentifiers({ name: 'Old', version: 3, containerData: 'r:/data' })).to.deep.equal(['Old']); + }); + + it('names nothing for an app with no synced component, or no spec', () => { + expect(mountParser.syncedComponentIdentifiers({ name: 'Old', version: 3, containerData: '/data' })).to.deep.equal([]); + expect(mountParser.syncedComponentIdentifiers(null)).to.deep.equal([]); + }); + }); + describe('gComponentIdentifiers', () => { it('names every g: component of a composed app, in compose order', () => { const spec = { diff --git a/tests/unit/peerFolderLiveness.test.js b/tests/unit/peerFolderLiveness.test.js index 14a6fdbd81..07a36a5925 100644 --- a/tests/unit/peerFolderLiveness.test.js +++ b/tests/unit/peerFolderLiveness.test.js @@ -50,7 +50,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); expect(probes(), 'what another node holds is not this peer\'s answer').to.equal(0); }); @@ -94,7 +94,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); sinon.assert.notCalled(axiosMock.get); }); @@ -108,7 +108,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); }); @@ -188,7 +188,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); sinon.assert.notCalled(axiosMock.get); }); @@ -334,7 +334,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); }); @@ -350,7 +350,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); }); @@ -394,7 +394,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], }); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 045dd2250b..111da98bd4 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -1163,8 +1163,9 @@ describe('syncthingFolderStateMachine tests', () => { // A read it cannot make is not a smaller claim, it is no claim - and the node // stands down on exactly that. Left up, the last good figure makes every peer - // defer to a node that is refusing to lead, and nobody seeds. - it('withdraws the published claim when it can no longer read what it holds', async () => { + // defer to a node that is refusing to lead, and nobody seeds. Recorded as + // unreadable rather than removed, so the folder is not one it has yet to look at. + it('records the folder unreadable, claiming nothing, when it can no longer read what it holds', async () => { globalStateMock.folderHoldings = new Map([ ['test-app', { bytes: 5821604997, newestModified: 200 }], ['other-app', { bytes: 900, newestModified: 100 }], @@ -1181,7 +1182,7 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); - expect(globalStateMock.folderHoldings.has('test-app'), 'an unreadable folder still claims its last good figure').to.equal(false); + expect(globalStateMock.folderHoldings.get('test-app'), 'an unreadable folder still claims its last good figure').to.deep.equal({ unreadable: true }); expect(globalStateMock.folderHoldings.has('other-app'), 'another folder\'s claim is not this folder\'s to withdraw').to.equal(true); }); @@ -2410,6 +2411,72 @@ describe('syncthingFolderStateMachine tests', () => { }); }; + // A holder that has not yet measured what it holds of the folder - an install its + // monitor has not reached - would read as holding nothing, and the field would fall + // to the address order over a holder that may hold the owner's data. So nobody + // seeds while one has not looked. This node's address sorts first and it holds + // nothing, so the address order would give it the seed. + describe('a holder that has not yet looked at the folder', () => { + const coldStartBeside = (peerAnswer) => { + mockParams.localSocketAddr = '10.0.0.1:16127'; + mockParams.appLocation.resolves([ + { ip: '10.0.0.1:16127', runningSince: 2000, broadcastedAt: 1000 }, + { ip: '10.0.0.2:16127', runningSince: 2000, broadcastedAt: 1000 }, + ]); + syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [] })); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 0, inSyncBytes: 0, state: 'idle', receiveOnlyChangedFiles: 0, + }); + axiosMock.get.resolves({ data: { data: { ready: true, folders: [], holding: {}, ...peerAnswer } } }); + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { + restarted: false, numberOfExecutions: 1, leaderStreak: 5, + }); + }; + + it('keeps any node from seeding until it has', async () => { + const count = sinon.spy(fluxEventBus, 'count'); + try { + coldStartBeside({ notLookedAt: ['test-app'] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'seeded over a holder that had not looked at what it holds').to.equal('receiveonly'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderNotLookedAt'); + } finally { + count.restore(); + } + }); + + it('a canary: the same holder, having looked and holding nothing, leaves the seed to the address order', async () => { + coldStartBeside({ notLookedAt: [] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + + it('is waited on for one of the app\'s other elected folders too', async () => { + coldStartBeside({ notLookedAt: ['sibling-app'] }); + mockParams.electedSiblings = ['sibling-app']; + // no node writes the app's other folder yet, so the election decides this one + appQueryServiceMock.holdsComponent.resolves(false); + globalStateMock.folderHoldings = new Map([['sibling-app', { bytes: 0, newestModified: 0 }]]); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('receiveonly'); + }); + + // A peer too old to say reads as having looked at everything, as it always has. + it('is not assumed of a peer that does not say', async () => { + coldStartBeside({}); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + }); + // Every claim is filed under the address the election list carries, because an // object key matches exactly where addresses compare tolerantly. Keyed from // anywhere else, this node stops finding its OWN claim as soon as the two From 8d9e0a949889dd0c324c6a260207b4ea2dd88202 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:19:57 +0100 Subject: [PATCH 281/369] test(harness): 51, nobody seeds while a holder has not yet measured what it holds, and then the holder seeds Co-Authored-By: Claude Opus 5.5 --- .../51-reconciler-syncthing-coldstart.js | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js b/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js index 3a3ce32b87..87c1965556 100644 --- a/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js +++ b/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js @@ -34,6 +34,13 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const subnet = getSubnetConfig(); +// The state machine's checkpoint before it measures what this node holds of a +// receive-only folder, keyed by folder id. +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +// Passes over a folder a node is counted deciding while another holder is held: +// more than the leadership confirmation needs, so a win would have been acted on. +const HELD_PASSES = 4; + async function isUp(client, appName) { const status = await getAppContainerStatus(client.container, appName); return !!(status && status.status.startsWith('Up')); @@ -52,6 +59,7 @@ describe('reconciler cold start - fresh multi-node placement, no seeded source', const gApp = `e2ecoldg${Date.now()}`; const claimApp = `e2ecoldclaim${Date.now()}`; const splitApp = `e2ecoldsplit${Date.now()}`; + const waitApp = `e2ecoldwait${Date.now()}`; const holders = [0, 1, 2]; // Asked rather than assumed. `holders[0]` is the lowest address only while the // list happens to be written in order, and it silently names the wrong node @@ -126,6 +134,51 @@ describe('reconciler cold start - fresh multi-node placement, no seeded source', // signed endpoint and ranks the field on the answers. A unit test reaches the // comparator with a claims object handed to it; only a fleet reaches it through a // node's own reading of its volume and a peer's reading of that node. + // THE CLAIM HAS TO HAVE BEEN MADE. A holder answers what it holds once its monitor + // has measured the folder; between its install and that pass it has measured + // nothing, and read then it holds nothing, so the field falls to the address order + // over the node holding the owner's data. The third holder - neither the + // address-order winner nor the one holding the data - is held just before it + // measures the folder, which is the window an install opens on every holder. + it('r: nobody seeds while a holder has not yet measured what it holds, and then the holder seeds', async function () { + this.timeout(300000); + const folder = `flux${waitApp}_${waitApp}`; + const unmeasured = holders.find((i) => i !== seedIndex && i !== claimHolder); + expect(unmeasured, 'no holder is left between the address-order winner and the data holder').to.not.equal(undefined); + const passes = () => env.clients[seedIndex].getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + + await pushImage(waitApp, 'v1'); + const spec = await buildSeedableSyncthingApp({ name: waitApp, mode: 'r' }); + await pinColdStart(holders, folder); + await pinHolding(claimHolder, folder, claimBytes); + await env.clients[unmeasured].holdCheckpoint(BEFORE_HOLDINGS, folder); + const from = env.clients[unmeasured].getLastEventId(); + try { + await installOnNodes(env, spec, holders); + await env.clients[unmeasured].waitForEvent('checkpoint:held', (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 120000, { afterId: from }); + + // The address-order winner decides the folder pass after pass while the third + // holder has not measured it - long enough to confirm a win it would act on. + const passesFrom = await passes(); + await waitFor(async () => (await passes()) >= passesFrom + HELD_PASSES, { + timeout: 120000, interval: 2000, label: `the address-order winner ran ${HELD_PASSES} passes over the folder`, + }); + expect( + await countUp(env, holders, waitApp), + 'a holder seeded while another had not yet measured what it holds', + ).to.equal(0); + } finally { + await env.clients[unmeasured].releaseCheckpoint(BEFORE_HOLDINGS, folder); + } + + await waitFor(async () => (await countUp(env, holders, waitApp)) >= 1, { + timeout: 90000, interval: 3000, label: 'a holder seeds once every holder has measured what it holds', + }); + await new Promise((r) => { setTimeout(r, 12000); }); + expect(await countUp(env, holders, waitApp)).to.equal(1); + expect(await isUp(env.clients[claimHolder], waitApp), 'the node holding the data did not seed').to.equal(true); + }); + it('r: the holder seeds, against the address the field would otherwise elect', async function () { this.timeout(180000); // The premise, asserted so a fixture that stops being divergent fails here rather From 56ddc31993225a7d6935dd09b116736a19ee54aa Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:22:27 +0100 Subject: [PATCH 282/369] test(harness): 2252, a fleet of two releases seeds a cold-start folder on the holder Co-Authored-By: Claude Opus 5.5 --- ...seeds-a-cold-start-folder-on-the-holder.js | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js diff --git a/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js b/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js new file mode 100644 index 0000000000..9cb482f759 --- /dev/null +++ b/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js @@ -0,0 +1,143 @@ +// base-image: FLUX_E2E_BASE_TAG +/* + * A fleet part way through an upgrade decides a cold-start seed across both releases. + * + * Three nodes take an r: app at once. The one holding the owner's data runs the released + * FluxOS, which ranks the field on its peers' signed holdings answers but does not read + * which folders a peer has yet to measure. The other two run this build: the one at the + * lowest address, which the address order would make the seed, and a third, held just + * before it measures the folder. + * + * While the third has not measured, the new node waits on it, and the released holder, + * reading the third as answering without a claim, falls to the address order and does + * not lead either: nobody seeds. Once it has measured, both rank the same field and the + * released holder seeds. It reaches that only by verifying the new nodes' answers, + * which carry a field it does not know: an answer it could not verify would read as + * unproven, which blocks its promotion for good. + * + * Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { getAppContainerStatus } from '../framework/container.js'; +import { resetSyncState, pinColdStart, pinHolding } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { syncthingSeedIndex } from '../framework/g-app-placement.js'; +import { buildSeedableSyncthingApp } from '../framework/seed-helper.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; +const subnet = getSubnetConfig(); +const HOLDERS = [0, 1, 2]; +// The holder the address order would pick last, so that seeding it is a decision the +// address order cannot also have produced. It runs the released FluxOS. +const DATA_HOLDER = [...HOLDERS].sort((a, b) => (subnet.nodeIp(a + 1) < subnet.nodeIp(b + 1) ? -1 : 1)).pop(); +const ADDRESS_WINNER = syncthingSeedIndex(HOLDERS); +const UNMEASURED = HOLDERS.find((i) => i !== ADDRESS_WINNER && i !== DATA_HOLDER); +const HOLDER_BYTES = 5821604997; +// The state machine's checkpoint before it measures what this node holds of a +// receive-only folder, keyed by folder id. +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +// Passes over the folder the new address-order winner is counted deciding while the +// third holder is held: more than the leadership confirmation needs, so a win would +// have been acted on. +const HELD_PASSES = 4; + +async function isUp(client, appName) { + const status = await getAppContainerStatus(client.container, appName); + return !!(status && status.status.startsWith('Up')); +} + +function step(what) { + console.log(`# 2252 ${new Date().toISOString()} ${what}`); +} + +describe('a mixed fleet seeds a cold-start folder on the holder', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2emixedseed${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const countUp = async () => (await Promise.all(HOLDERS.map((i) => isUp(env.clients[i], appName)))).filter(Boolean).length; + let releasedVersion; + let thisVersion; + let parked; + let passesWhileHeld; + let upWhileHeld; + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, nodes: 10, tickerAutostart: false, nodeImageTags: { [DATA_HOLDER]: BASE_TAG }, + }); + if (BASE_TAG === env.nodeImageTag(ADDRESS_WINNER)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG'); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + releasedVersion = (await env.clients[DATA_HOLDER].getVersion())?.data; + thisVersion = (await env.clients[ADDRESS_WINNER].getVersion())?.data; + + await pushImage(appName, 'v1'); + const spec = await buildSeedableSyncthingApp({ name: appName, mode: 'r' }); + await pinColdStart(HOLDERS, folder); + await pinHolding(DATA_HOLDER, folder, HOLDER_BYTES); + + const passes = () => env.clients[ADDRESS_WINNER].getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + await env.clients[UNMEASURED].holdCheckpoint(BEFORE_HOLDINGS, folder); + const from = env.clients[UNMEASURED].getLastEventId(); + try { + step('installing on all three holders, the third held before it measures the folder'); + await installOnNodes(env, spec, HOLDERS); + parked = Boolean(await env.clients[UNMEASURED].waitForEvent( + 'checkpoint:held', (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 120000, { afterId: from }, + ).catch(() => null)); + const passesFrom = await passes(); + await waitFor(async () => (await passes()) >= passesFrom + HELD_PASSES, { + timeout: 120000, interval: 2000, label: `the address-order winner ran ${HELD_PASSES} passes over the folder`, + }).catch(() => {}); + passesWhileHeld = (await passes()) - passesFrom; + upWhileHeld = await countUp(); + } finally { + await env.clients[UNMEASURED].releaseCheckpoint(BEFORE_HOLDINGS, folder); + } + step('released'); + }); + + after(async function () { + this.timeout(60000); + await resetSyncState().catch(() => {}); + await resetFdm().catch(() => {}); + await env?.teardown(); + }); + + it('runs the data holder on another release than the other two', function () { + expect(releasedVersion, 'the data holder answered no version').to.be.a('string'); + expect(releasedVersion, 'every holder runs the same release, so nothing below is mixed').to.not.equal(thisVersion); + }); + + it('held the third holder before it measured the folder, while the address-order winner decided it', function () { + expect(parked, 'the third holder was never held, so nothing below was tested').to.equal(true); + expect(passesWhileHeld, 'the address-order winner did not decide the folder while the third was held').to.be.at.least(HELD_PASSES); + }); + + it('seeds nothing while a holder of this build has not measured what it holds', function () { + expect(upWhileHeld, 'a holder seeded while another had not yet measured what it holds').to.equal(0); + }); + + it('seeds the released data holder once every holder has measured, and only it', async function () { + this.timeout(180000); + await waitFor(async () => (await countUp()) >= 1, { + timeout: 120000, interval: 3000, label: 'a holder seeds once every holder has measured what it holds', + }); + await new Promise((r) => { setTimeout(r, 12000); }); + expect(await countUp(), 'more than one holder seeded the folder').to.equal(1); + expect(await isUp(env.clients[DATA_HOLDER], appName), 'the released data holder did not seed').to.equal(true); + }); +}); From 9a9d5178cdd68a4fb7e2a2a583c3c241ceb916b9 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:51:40 +0100 Subject: [PATCH 283/369] test(harness): a mixed fleet runs different FluxOS code on each image, checked on the running nodes createTestEnv hashes the FluxOS code inside one running node of each image tag and refuses a fleet whose tags hold the same code; upgradeNode refuses a move that leaves a node's code unchanged. Every mixed-version suite relies on these checks rather than on version numbers or tag names. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/test-env.js | 33 +++++++++++++++++++ ...seeds-a-cold-start-folder-on-the-holder.js | 9 ----- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 5bcfbb20ef..f944b7e142 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -244,6 +244,34 @@ const UPGRADE_FILES = ['package.json', 'ZelBack/pinEnvironment.js', 'ZelBack/con // The new build's per-node config files and the script that writes the overlay from them. const UPGRADE_HARNESS = ['test-infra/config', 'test-infra/write-node-config.sh']; +// A digest of the FluxOS code a running node holds: the path and content of every +// file the process loads, so two builds of the same code agree whatever their +// version, tag or file times. +async function fluxosCodeDigest(container) { + const paths = [...UPGRADE_DIRS, ...UPGRADE_FILES].join(' '); + const res = await execInContainer(container, `cd /flux && find ${paths} -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum`); + const digest = res.stdout.trim().split(/\s+/)[0]; + if (res.exitCode !== 0 || !/^[0-9a-f]{64}$/.test(digest)) throw new Error(`cannot read the FluxOS code of ${container.getId()}: ${res.output}`); + return digest; +} + +// A fleet that names more than one image runs more than one FluxOS: every tag's +// nodes hold code no other tag's nodes hold. +async function assertTagsRunOtherCode(fluxNodes) { + const byTag = new Map(); + for (const node of fluxNodes) { + if (node.container && !byTag.has(node.imageTag)) byTag.set(node.imageTag, node.container); + } + if (byTag.size < 2) return; + const tags = [...byTag.keys()]; + const digests = await Promise.all(tags.map((tag) => fluxosCodeDigest(byTag.get(tag)))); + const tagOf = new Map(); + digests.forEach((digest, i) => { + if (tagOf.has(digest)) throw new Error(`node images ${nodeImageRef(tagOf.get(digest))} and ${nodeImageRef(tags[i])} hold the same FluxOS code, so the fleet is not mixed`); + tagOf.set(digest, tags[i]); + }); +} + async function dockerCp(from, to) { const res = await dockerCli(['cp', from, to], 180000); if (!res.ok) throw new Error(`docker cp ${from} ${to}: ${res.stderr.trim()}`); @@ -1885,6 +1913,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod } } containers.fluxNodes = fluxNodes; + await assertTagsRunOtherCode(fluxNodes); for (const n of fluxNodes) { if (!n.container) { @@ -2029,6 +2058,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod if (!tag) throw new Error('upgradeNode: the image tag to run is required'); await assertNodeImageStamped(tag); const nodeId = node.container.getId(); + const codeBefore = await fluxosCodeDigest(node.container); const created = await dockerCli(['create', nodeImageRef(tag)], 60000); if (!created.ok) throw new Error(`upgradeNode: cannot read ${nodeImageRef(tag)}: ${created.stderr.trim()}`); const sourceId = created.stdout.trim(); @@ -2062,6 +2092,9 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod await dockerCli(['rm', '-f', sourceId], 60000); rmSync(host, { recursive: true, force: true }); } + if (await fluxosCodeDigest(node.container) === codeBefore) { + throw new Error(`upgradeNode: ${nodeImageRef(tag)} holds the FluxOS code node ${index} already ran, so nothing was upgraded`); + } await restartFluxos(node.container, { readyTimeoutMs }); node.imageTag = tag; }, diff --git a/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js b/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js index 9cb482f759..9e7f27318b 100644 --- a/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js +++ b/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js @@ -63,8 +63,6 @@ describe('a mixed fleet seeds a cold-start folder on the holder', function () { const appName = `e2emixedseed${Date.now()}`; const folder = `flux${appName}_${appName}`; const countUp = async () => (await Promise.all(HOLDERS.map((i) => isUp(env.clients[i], appName)))).filter(Boolean).length; - let releasedVersion; - let thisVersion; let parked; let passesWhileHeld; let upWhileHeld; @@ -81,8 +79,6 @@ describe('a mixed fleet seeds a cold-start folder on the holder', function () { await bootAndPeer(env); await resetFdm(); await resetSyncState(); - releasedVersion = (await env.clients[DATA_HOLDER].getVersion())?.data; - thisVersion = (await env.clients[ADDRESS_WINNER].getVersion())?.data; await pushImage(appName, 'v1'); const spec = await buildSeedableSyncthingApp({ name: appName, mode: 'r' }); @@ -117,11 +113,6 @@ describe('a mixed fleet seeds a cold-start folder on the holder', function () { await env?.teardown(); }); - it('runs the data holder on another release than the other two', function () { - expect(releasedVersion, 'the data holder answered no version').to.be.a('string'); - expect(releasedVersion, 'every holder runs the same release, so nothing below is mixed').to.not.equal(thisVersion); - }); - it('held the third holder before it measured the folder, while the address-order winner decided it', function () { expect(parked, 'the third holder was never held, so nothing below was tested').to.equal(true); expect(passesWhileHeld, 'the address-order winner did not decide the folder while the third was held').to.be.at.least(HELD_PASSES); From 7da14ed4a8896bb7d47fcd4357669b3ce3a0cacf Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 15:03:09 +0100 Subject: [PATCH 284/369] test(harness): 2214's holder-first case holds both contenders and releases the holder first Co-Authored-By: Claude Opus 5.5 --- ...214-two-holders-that-see-each-other-never-both-seed.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js b/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js index a13e34fe5b..5ad0113a56 100644 --- a/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js +++ b/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js @@ -168,9 +168,13 @@ describe('two holders that can reach each other never both seed', function () { it('seeds once, on the holder, when the holder has decided before the lowest address reads it', async function () { this.timeout(600000); - const app = await placeSplitApp('holderfirst', [[lowest, BEFORE_DECISION]]); - await heldAt(lowest, BEFORE_DECISION, app); + // Both held, so the lowest address is at its decision before the holder seeds: + // left to run, the holder seeds before the lowest address has confirmed its + // election, and the lowest address never reaches the decision at all. + const app = await placeSplitApp('holderfirst', contenders.map((i) => [i, BEFORE_DECISION])); + await Promise.all(contenders.map((i) => heldAt(i, BEFORE_DECISION, app))); + await release(holder, BEFORE_DECISION, app); await waitFor(async () => (await runners(app.name)).includes(holder), { timeout: 120000, interval: 1000, label: 'the holder seeds while the lowest address is held', }); From d795a51367952f11cdcc9769d305b1fec47d568d Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 18:31:47 +0100 Subject: [PATCH 285/369] test(harness): a g: app's holders are placed empty, and a folder pinned synced gets its bytes from the keeper Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/reconciler-suite.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/framework/reconciler-suite.js b/test-infra/runner/framework/reconciler-suite.js index 6e7ba2d8a5..891adf1320 100644 --- a/test-infra/runner/framework/reconciler-suite.js +++ b/test-infra/runner/framework/reconciler-suite.js @@ -228,12 +228,12 @@ export async function placeGAppInOrder(env, app, { const installAfter = env.clients[i].getLastEventId(); // eslint-disable-next-line no-await-in-loop await installOnNodes(env, app, [i]); - // Wait out the sync layer's first-run reset before writing disk data, or the - // reset deletes it - a folder pinned synced later must hold what its index claims. + // Every holder enters the election empty, as a newborn app's holders do: the + // sync layer's first-run reset has cleared the volume before the next holder is + // placed. A folder a suite pins synced later gets its bytes from the synced-data + // keeper. // eslint-disable-next-line no-await-in-loop await waitForReconcileActuated(env.clients[i], identifier, 'dataCleared', 60000, { afterId: installAfter }); - // eslint-disable-next-line no-await-in-loop - await seedSyncScopedData(env, app.spec.name, i); // WAITED FOR, not slept past. The ordering these placements exist to create // is the runningSince stamps, and a fixed gap produces it only while every From 7740a73da73d77c4e82da68da1e0a1a68c46ffc7 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 18:44:23 +0100 Subject: [PATCH 286/369] test(harness): 2209's partial-restore case puts the primary's data in place before it locks the volume Co-Authored-By: Claude Opus 5.5 --- .../tests/2209-a-single-writer-folder-type-has-one-writer.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js b/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js index d4ee10cd22..0b2eae489a 100644 --- a/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js +++ b/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js @@ -393,7 +393,9 @@ describe('a single-writer folder type has one writer', function () { `rm -rf /tmp/s-${a.appName} && mkdir -p /tmp/s-${a.appName} && printf 'restored\\n' > /tmp/s-${a.appName}/restored.txt ` + `&& mkdir -p ${dir}/backup/local && tar -czf ${archive} -C /tmp/s-${a.appName} .`); expect(staged.exitCode, `fixture: staging the archive failed: ${staged.output}`).to.equal(0); - // The clear fails the way a volume gone read-only under an ext4 error does. + // The clear fails the way a volume gone read-only under an ext4 error does: the + // primary's data cannot be deleted. + await seedSyncScopedData(env, a.appName, a.primary); const lock = await execInContainer(client.container, `chattr +i ${dir}/appdata`); expect(lock.exitCode, `fixture: could not make appdata immutable: ${lock.output}`).to.equal(0); const from = client.getLastEventId(); From 4bf8a660a4746e69f09d8c2c8444f489b5208ee1 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 19:22:47 +0100 Subject: [PATCH 287/369] test(harness): 68's senior-standby case holds the seed before it measures until every later holder has Co-Authored-By: Claude Opus 5.5 --- .../tests/68-masterslave-election-order.js | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/68-masterslave-election-order.js b/test-infra/runner/tests/68-masterslave-election-order.js index 090ecc352c..6337e9c434 100644 --- a/test-infra/runner/tests/68-masterslave-election-order.js +++ b/test-infra/runner/tests/68-masterslave-election-order.js @@ -52,6 +52,10 @@ const BEFORE_START = 'masterSlave:beforeStart'; // writing it, keyed by folder id. const BEFORE_FOLDER_WRITE = 'syncthing:beforeFolderWrite'; +// The syncthing monitor's checkpoint before a receiving folder's holdings are +// measured, keyed by folder id. +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; + async function isUp(client, appName) { const status = await getAppContainerStatus(client.container, appName); return !!(status && status.status.startsWith('Up')); @@ -651,9 +655,28 @@ describe('primary election under a divergent placement order', function () { // // The seed is placed first, so the genesis winner is election index 0 and both // standbys are below it: each has a turn to book. + // + // The seed is held before it measures the folder until every later holder has, + // so it cannot win on its own before the others are placed: the seed is decided + // over the whole field, and the address order decides it because no holder + // holds anything. const deployedFrom = holders.map((i) => env.clients[i].getLastEventId()); - await deploy(lossApp, placementOrderWithSeedAt(holders, 0)); const folder = folderOf(lossApp); + const others = holders.filter((i) => i !== seedIndex); + const waitedOnSeed = (i) => env.clients[i].getDecisionCount('syncthing:seedDecision', folder, 'holderNotLookedAt'); + await env.clients[seedIndex].holdCheckpoint(BEFORE_HOLDINGS, folder); + try { + await deploy(lossApp, placementOrderWithSeedAt(holders, 0)); + await env.clients[seedIndex].waitForEvent('checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 120000, { afterId: deployedFrom[holders.indexOf(seedIndex)] }); + // A holder that has measured the folder waits on the seed, which has not. + await Promise.all(others.map((i) => waitFor(async () => (await waitedOnSeed(i)) > 0, { + timeout: 120000, interval: 1000, label: `holder ${i} measured the folder and waits on the seed`, + }))); + } finally { + await env.clients[seedIndex].releaseCheckpoint(BEFORE_HOLDINGS, folder) + .catch((err) => console.warn(`cleanup: checkpoint release failed: ${err.message}`)); + } await Promise.all(holders.map((i) => setSynced({ ip: subnet.nodeIp(i + 1), folder }))); await Promise.all(holders.map((i, k) => env.clients[i].waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 180000, { afterId: deployedFrom[k] }))); From 40e27f89746e83de804266510c9d77f1d17db02b Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 19:37:23 +0100 Subject: [PATCH 288/369] test(harness): 68's senior-standby case pins the folders synced only once the seed election has picked Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/tests/68-masterslave-election-order.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test-infra/runner/tests/68-masterslave-election-order.js b/test-infra/runner/tests/68-masterslave-election-order.js index 6337e9c434..dcb1c7d9fc 100644 --- a/test-infra/runner/tests/68-masterslave-election-order.js +++ b/test-infra/runner/tests/68-masterslave-election-order.js @@ -677,6 +677,13 @@ describe('primary election under a divergent placement order', function () { await env.clients[seedIndex].releaseCheckpoint(BEFORE_HOLDINGS, folder) .catch((err) => console.warn(`cleanup: checkpoint release failed: ${err.message}`)); } + // The genesis winner is the holder the seed election picks. A folder pinned + // synced before it has picked lets the primary election start index 0 instead. + const seeded = async () => (await Promise.all(holders.map((i) => env.clients[i] + .getDecisionCount('syncthing:seedDecision', folder, 'seeded')))).reduce((sum, n) => sum + n, 0); + await waitFor(async () => (await seeded()) > 0, { + timeout: 120000, interval: 1000, label: 'a holder decided to seed the folder', + }); await Promise.all(holders.map((i) => setSynced({ ip: subnet.nodeIp(i + 1), folder }))); await Promise.all(holders.map((i, k) => env.clients[i].waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 180000, { afterId: deployedFrom[k] }))); From 098fdf4aed1fe239c9e5d67d5712a58bd2a3d496 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 19:47:50 +0100 Subject: [PATCH 289/369] fix(syncthing): each receive-only folder has one state from its install to its removal, and a peer waits only on one not yet looked at A peer deciding a cold-start seed waits while a holder lists the folder as not looked at. Each receive-only folder of an installed app now has exactly one state (appMonitoring/folderStates), written where its fact becomes known: not looked at on install, on a volume recreate and on the first pass after a restart; measured or no claim when a pass measures it; held out when a pass holds it out because its volume is not safe to act on; no claim once it is promoted, and on a ready standby that holds no measurement; removed with the volume. A folder held out for its mount, a demoted ex-primary and a standby that was running at a FluxOS restart each read as no claim, so no peer waits on them. The promoted-folders answer is built from that state in memory: the endpoint reads no database and decrypts nothing per request. Its fields are unchanged. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 4 +- .../src/services/appLifecycle/appInstaller.js | 4 + .../services/appLifecycle/appUninstaller.js | 3 +- .../services/appMonitoring/folderStates.js | 155 ++++++++++++++++++ .../syncthingFolderStateMachine.js | 19 +-- .../appMonitoring/syncthingFolderWrites.js | 6 +- .../appMonitoring/syncthingMonitor.js | 9 + .../src/services/appQuery/appQueryService.js | 43 +---- ZelBack/src/services/utils/globalState.js | 14 +- tests/unit/advancedWorkflows.test.js | 49 ++++-- tests/unit/appInstaller.test.js | 22 +++ tests/unit/appQueryService.test.js | 70 ++++---- tests/unit/appUninstaller.test.js | 12 +- tests/unit/folderStates.test.js | 120 ++++++++++++++ .../unit/syncthingFolderStateMachine.test.js | 46 ++++-- tests/unit/syncthingMonitor.test.js | 105 +++++++++++- 16 files changed, 548 insertions(+), 133 deletions(-) create mode 100644 ZelBack/src/services/appMonitoring/folderStates.js create mode 100644 tests/unit/folderStates.test.js diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 45492ac790..9854e662a5 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -55,6 +55,7 @@ const appsRuntimeState = require('../appManagement/appsRuntimeState'); const { stopAppMonitoring } = require('../appManagement/appInspector'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const globalState = require('../utils/globalState'); +const folderStates = require('../appMonitoring/folderStates'); const appNetworkLinker = require('./appNetworkLinker'); const { Privilege, authOf } = require('../utils/privileges'); const { AsyncLock } = require('../utils/asyncLock'); @@ -678,7 +679,7 @@ async function createAppVolume(appSpecifications, appName, isComponent, res) { // Same dead incarnation, and this claim is published: peers rank a seed on what // each node says it holds, so the old volume's figures describe the one being // replaced and can win the election for a volume that is about to be empty. - globalState.folderHoldings?.delete(appId); + folderStates.recreated(appId); globalState.seedMarks.delete(appId); await execAsRoot('fallocate', ['-l', `${appSpecifications.hdd}G`, volumeFile]); const allocateSpace2 = { @@ -1153,6 +1154,7 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name} in soft registration. Database insert returned undefined - likely duplicate key error or database failure. Aborting soft registration to prevent orphaned Docker containers.`); } log.info(`Database entry created for ${appSpecifications.name} BEFORE Docker container creation (soft registration)`); + folderStates.installed(appSpecifications); const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; const cpuTier = `cpu${tier}`; diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index d548eb90ac..6d6eb2dfc6 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -27,6 +27,7 @@ const imageVerifier = require('../utils/imageVerifier'); const registryCredentialHelper = require('../utils/registryCredentialHelper'); const upnpService = require('../upnpService'); const globalState = require('../utils/globalState'); +const folderStates = require('../appMonitoring/folderStates'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const mountParser = require('../utils/mountParser'); @@ -542,6 +543,9 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name}. Database insert returned undefined - likely duplicate key error or database failure. Aborting installation to prevent orphaned Docker containers.`); } log.info(`Database entry created for ${appSpecifications.name} BEFORE Docker container creation`); + // Installed from here: a peer deciding a cold-start seed waits on this node + // until its monitor has measured what it holds. + folderStates.installed(appSpecifications); claimed = true; const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index cfbdb7e90b..8e3305900f 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -7,6 +7,7 @@ const messageHelper = require('../messageHelper'); const dockerService = require('../dockerService'); const dbHelper = require('../dbHelper'); const globalState = require('../utils/globalState'); +const folderStates = require('../appMonitoring/folderStates'); const log = require('../../lib/log'); const { localAppsInformation, globalAppsInformation, globalAppsMessages, ANNOUNCE_CYCLE_WAIT_MS, @@ -155,7 +156,7 @@ async function stopSyncthingAndCleanup(monitoredName, appId, res) { // and one kept past the removal offers a volume this node no longer has - // outranking a node that still holds the app and seeding an empty folder in its // place. - globalState.folderHoldings?.delete(appId); + folderStates.removed(appId); globalState.seedMarks.delete(appId); try { diff --git a/ZelBack/src/services/appMonitoring/folderStates.js b/ZelBack/src/services/appMonitoring/folderStates.js new file mode 100644 index 0000000000..f74b6a642a --- /dev/null +++ b/ZelBack/src/services/appMonitoring/folderStates.js @@ -0,0 +1,155 @@ +const globalState = require('../utils/globalState'); +const mountParser = require('../utils/mountParser'); +const dockerService = require('../dockerService'); + +// What this node tells its peers about each receive-only folder of an installed app, +// for a cold-start seed: which holder starts the app from what it holds. +// +// Every such folder has exactly one entry from its install to its removal, so a +// missing entry means only that the folder is not installed here. A peer about to +// seed waits on a folder this node has not looked at, ranks one it has measured, +// and reads every other as no claim. +// +// Written where each fact becomes known: the install and a volume recreate, the +// monitor pass, a promotion, the removal. Nothing else removes an entry. + +const FolderState = Object.freeze({ + // Installed, and no monitor pass has reached it since. + NOT_LOOKED_AT: 'notLookedAt', + // { bytes, newestModified } of what this node holds that the cluster's index does not. + MEASURED: 'measured', + // Looked at, with nothing to rank: its holdings could not be read, it has sent + // everything it held, or it is a ready standby. + NO_CLAIM: 'noClaim', + // Held out of the monitor pass because its volume is not safe to act on. + HELD_OUT: 'heldOut', +}); + +// What a peer is sent for a folder this node has looked at and cannot claim +// anything of. Any value that is not a rankable claim reads as no claim. +const NO_CLAIM_ANSWER = Object.freeze({ unreadable: true }); + +/** + * The folder ids of an app's components that sync receive-only (g: and r:). + * @param {object} appSpec Installed app specification, decrypted. + * @returns {string[]} + */ +function receiveOnlyFolderIds(appSpec) { + const components = appSpec.version <= 3 + ? [{ identifier: appSpec.name, containerData: appSpec.containerData }] + : (appSpec.compose || []).map((component) => ({ identifier: `${component.name}_${appSpec.name}`, containerData: component.containerData })); + return components + .filter(({ containerData }) => ['g', 'r'].includes(mountParser.getComponentSyncMode(containerData))) + .map(({ identifier }) => dockerService.getAppIdentifier(identifier)); +} + +/** + * Records the receive-only folders of an installed app that have no entry as not + * looked at. A folder that already has one keeps it: its volume is the one it + * describes. + * @param {object} appSpec Installed app specification, decrypted. + */ +function installed(appSpec) { + receiveOnlyFolderIds(appSpec) + .filter((folderId) => !globalState.folderStates.has(folderId)) + .forEach((folderId) => globalState.folderStates.set(folderId, { state: FolderState.NOT_LOOKED_AT })); +} + +/** + * Records a folder whose volume has been replaced by an empty one as not looked at. + * @param {string} folderId + */ +function recreated(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.NOT_LOOKED_AT }); +} + +/** + * Removes a folder whose volume has been deleted. + * @param {string} folderId + */ +function removed(folderId) { + globalState.folderStates.delete(folderId); +} + +/** + * Records what a monitor pass measured of a folder. + * @param {string} folderId + * @param {{bytes: number, newestModified: number}|null} holdings null when they could not be read. + */ +function measured(folderId, holdings) { + globalState.folderStates.set(folderId, holdings + ? { state: FolderState.MEASURED, bytes: holdings.bytes, newestModified: holdings.newestModified } + : { state: FolderState.NO_CLAIM }); +} + +/** + * Records a folder the monitor pass held out because its volume is not safe to act on. + * @param {string} folderId + */ +function heldOut(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.HELD_OUT }); +} + +/** + * Records a folder that has turned writable: everything this node held of it is + * published, so it has nothing left to claim. + * @param {string} folderId + */ +function sending(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.NO_CLAIM }); +} + +/** + * Records a folder the monitor pass found a ready standby. A measurement it holds + * stands; any other state becomes no claim, because a ready standby is not measured. + * @param {string} folderId + */ +function settled(folderId) { + if (globalState.folderStates.get(folderId)?.state === FolderState.MEASURED) return; + globalState.folderStates.set(folderId, { state: FolderState.NO_CLAIM }); +} + +/** + * What this node claims of a folder, as a peer reads it: a rankable claim, the no-claim + * answer, or undefined for a folder it has not looked at or does not hold. + * @param {string} folderId + * @returns {{bytes: number, newestModified: number}|object|undefined} + */ +function claimOf(folderId) { + const entry = globalState.folderStates.get(folderId); + if (!entry || entry.state === FolderState.NOT_LOOKED_AT) return undefined; + if (entry.state === FolderState.MEASURED) return { bytes: entry.bytes, newestModified: entry.newestModified }; + return NO_CLAIM_ANSWER; +} + +/** + * The receive-only part of the promoted-folders answer. A writable folder is in + * neither: a peer reads it from the folders this node sends. + * @param {Set} writable The folders this node holds writable. + * @returns {{holding: object, notLookedAt: string[]}} + */ +function answer(writable) { + const holding = {}; + const notLookedAt = []; + globalState.folderStates.forEach((entry, folderId) => { + if (writable.has(folderId)) return; + if (entry.state === FolderState.NOT_LOOKED_AT) notLookedAt.push(folderId); + else holding[folderId] = claimOf(folderId); + }); + return { holding, notLookedAt }; +} + +module.exports = { + FolderState, + NO_CLAIM_ANSWER, + answer, + claimOf, + heldOut, + installed, + measured, + receiveOnlyFolderIds, + recreated, + removed, + sending, + settled, +}; diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 753b042ef5..fae46e5345 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -28,6 +28,7 @@ const { const { isPathMounted } = require('../utils/volumeService'); const globalState = require('../utils/globalState'); +const folderStates = require('./folderStates'); const { isSyncedRootName } = require('../appSystem/volumeReservedNames'); const primaryRole = require('../appLifecycle/primaryRole'); const { componentStateOnOtherHolders } = require('./peerComponent'); @@ -649,10 +650,6 @@ function lowestIpHolder(allPeersList) { * @param {object} claim - a candidate's { bytes, newestModified }, or nothing * @returns {boolean} */ -// What a folder's holdings are recorded as when this node cannot read them: no claim, -// and not a folder it has yet to look at. -const UNREADABLE_HOLDINGS = Object.freeze({ unreadable: true }); - function isRankableClaim(claim) { return Number.isFinite(claim?.bytes) && Number.isFinite(claim.newestModified); } @@ -1489,12 +1486,11 @@ async function handleReceiveOnlyTransition(params) { // not a smaller claim, it is no claim - and standDownOnUnknown below refuses to lead // on exactly that. Left up, the last good figure says 5.8 GB to every peer while this // node refuses to act on it: they rank it first and defer, it defers to its own - // unknown, and nobody seeds for as long as the read keeps failing. Recorded as - // unreadable, a peer reads it as no claim, which drops the field to the address - // order - the case that already exists for a peer too old to answer at all - and the - // folder is not one this node has yet to look at (promotedFolderHoldings). - if (!globalState.folderHoldings) globalState.folderHoldings = new Map(); - globalState.folderHoldings.set(appId, holdings ?? UNREADABLE_HOLDINGS); + // unknown, and nobody seeds for as long as the read keeps failing. Recorded as no + // claim, it drops the field to the address order - the case that already exists for + // a peer too old to answer at all - and the folder is not one this node has yet to + // look at. + folderStates.measured(appId, holdings); // Designated-leader election, debounced: require leadership to hold for // LEADER_CONFIRM_COUNT consecutive cycles, so a single transient peer-visibility blip // doesn't flip a follower to leader. @@ -1531,7 +1527,7 @@ async function handleReceiveOnlyTransition(params) { // decides where the answer comes from and nothing else - keyed from anywhere but // the list, a node stops finding its own claim the moment the two spellings of // its address differ, and the ranking degrades to the address order silently. - const ownClaim = electedAppClaim(holdings, electedSiblings.map((sibling) => globalState.folderHoldings?.get(sibling))); + const ownClaim = electedAppClaim(holdings, electedSiblings.map((sibling) => folderStates.claimOf(sibling))); const claims = {}; // Candidates that hold this app and have not yet measured what they hold of it - an // install their monitor has not reached. Ranked now, one would read as holding @@ -2101,6 +2097,7 @@ async function manageFolderSyncState(params) { } // Default case - ensure container is running + folderStates.settled(appId); await ensureContainerRunning(appId, containerDataFlags); if (containerDataFlags.includes('g')) { // The ready mark the election reads has to outlive this pass. diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js b/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js index 453bb9f3db..b70d069bc3 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js @@ -2,6 +2,7 @@ const log = require('../../lib/log'); const serviceHelper = require('../serviceHelper'); const syncthingService = require('../syncthingService'); const globalState = require('../utils/globalState'); +const folderStates = require('./folderStates'); const fluxEventBus = require('../utils/fluxEventBus'); const { appsFolder } = require('../utils/appConstants'); const { OWNED_FOLDER_SETTINGS } = require('./syncthingMonitorHelpers'); @@ -81,9 +82,8 @@ function recordType(folderId, type) { writable?.add(folderId); // What a folder holds that the cluster's index does not is a receive-only // question, and promotion answers it: everything this node holds is now - // published. Dropped rather than zeroed - absent is what a peer reads as - // "not a receive-only holder". - globalState.folderHoldings?.delete(folderId); + // published. + folderStates.sending(folderId); // A folder that sends has been seeded; peers read that from the folder itself. globalState.seedMarks.delete(folderId); } else { diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 793f986b19..f2b04ebb1e 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -38,6 +38,7 @@ const volumeService = require('../utils/volumeService'); const appTamperingDetectionService = require('../appTamperingDetectionService'); const appReconciler = require('./appReconciler'); const syncthingFolderWrites = require('./syncthingFolderWrites'); +const folderStates = require('./folderStates'); const primaryRole = require('../appLifecycle/primaryRole'); const { manageFolderSyncState, @@ -351,6 +352,7 @@ async function processContainerData(params) { if (!markerReady) { log.warn(`processContainerData - ${appId} volume not mounted; skipping syncthing configuration this cycle`); fluxEventBus.count('syncthing:folderPass', appId, 'volumeNotMounted'); + folderStates.heldOut(appId); return; } @@ -377,6 +379,7 @@ async function processContainerData(params) { log.warn(`processContainerData - ${appId} locations could not be read (${error.message}); leaving the folder as it is this cycle`); fluxEventBus.count('syncthing:folderPass', appId, 'locationsUnread'); unsafeFolderIds.add(appId); + folderStates.heldOut(appId); return; } locations = sortAndFilterLocations(locations, localSocketAddr); @@ -582,6 +585,11 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // missing" by it, and the sweep at the end deletes by it. const ownerIds = syncingFolderOwnerIds(appsInstalled.data); + // Every receive-only folder of an installed app has a state before this node first + // answers its peers below. After a restart the states start empty while the volumes + // still hold their data, so each folder is not looked at until this pass measures it. + appsInstalled.data.forEach((installedApp) => folderStates.installed(installedApp)); + // Get required IDs and configurations const localDeviceId = await syncthingService.getDeviceId(); if (!localDeviceId) { @@ -777,6 +785,7 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { const unreadableIds = new Set(unreadableFolderEntries.map(({ appId }) => appId)); for (const { appId, reason } of unmountedApps) { unsafeFolderIds.add(appId); + folderStates.heldOut(appId); // The container is the other writer over the bad mount, and holding it // asks nothing of syncthing: a component that syncs the folder, or one // whose spec cannot be read to say it does not, is held before the diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index a77c7faadd..6f1d1a0ae8 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -17,6 +17,7 @@ const { socketAddressesMatch } = require('../utils/socketAddressUtils'); const networkStateService = require('../networkStateService'); const peerIdentityService = require('../peerIdentityService'); const primaryRoleChanges = require('../appLifecycle/primaryRoleChanges'); +const folderStates = require('../appMonitoring/folderStates'); const syncthingService = require('../syncthingService'); const log = require('../../lib/log'); @@ -823,35 +824,18 @@ async function callerIsFluxnode(body) { * The answer is signed over the challenge the request body carries, when it * carries one. * - * @param {object} req Request. - * @param {object} res Response. * `seeding` is where this node stands in deciding which holder seeds each folder at a * cold start (globalState.seedMarks); it carries the same figures as `holding`, so it * goes only where `holding` does. * - * `notLookedAt` is every synced folder of an installed app this node has neither - * measured nor made writable: an install its monitor has not yet reached. A peer about - * to seed waits on them, so a holder that has not looked is never read as holding - * nothing. A node that cannot read its installed apps cannot say, and answers not ready. + * `notLookedAt` is every receive-only folder of an installed app this node has not yet + * looked at (folderStates). A peer about to seed waits on them, so a holder that has + * not looked is never read as holding nothing. * + * @param {object} req Request. + * @param {object} res Response. * @returns {object} Message carrying { ready, folders, notLookedAt, holding, seeding }. */ -/** - * The synced folders of the apps installed here that this node has neither measured - * nor made writable. - * @param {Set} writable The folders this node holds writable. - * @param {Map|null} held The folders it has measured. - * @returns {Promise} - * @throws {Error} When the installed apps cannot be read. - */ -async function foldersNotLookedAt(writable, held) { - const { readable } = await decryptEnterpriseApps(await installedAppSpecs()); - return readable - .flatMap((app) => mountParser.syncedComponentIdentifiers(app)) - .map((identifier) => dockerService.getAppIdentifier(identifier)) - .filter((folder) => !writable.has(folder) && !held?.has(folder)); -} - async function promotedFolderHoldings(req, res) { try { // eslint-disable-next-line global-require @@ -880,24 +864,15 @@ async function promotedFolderHoldings(req, res) { const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); const entitled = authorized === true || await callerIsFluxnode(body); - const held = globalState.folderHoldings; - let ready = ids !== null; - let notLookedAt = []; - if (ready) { - try { - notLookedAt = await foldersNotLookedAt(ids, held); - } catch (error) { - log.warn(`promotedFolderHoldings - the installed apps could not be read, so which folders are measured cannot be said; answering not ready: ${error.message}`); - ready = false; - } - } + const ready = ids !== null; + const { holding, notLookedAt } = ready ? folderStates.answer(ids) : { holding: {}, notLookedAt: [] }; const seal = peerIdentityService.answerSealer(peerIdentityService.AnswerPurpose.PROMOTED_FOLDERS, body); const response = messageHelper.createDataMessage(await seal({ ready, folders: ids === null ? [] : [...ids], notLookedAt, ...(entitled ? { - holding: held === null ? {} : Object.fromEntries(held), + holding, seeding: Object.fromEntries(globalState.seedMarks), } : {}), })); diff --git a/ZelBack/src/services/utils/globalState.js b/ZelBack/src/services/utils/globalState.js index 53bad937be..64e0c26125 100644 --- a/ZelBack/src/services/utils/globalState.js +++ b/ZelBack/src/services/utils/globalState.js @@ -177,12 +177,11 @@ const fluxRemovedContainers = new Set(); // once. Same null-is-no-opinion convention appReconciler's controllerDesired uses. let promotedFolderIds = null; -// What each receive-only folder on this node holds that the cluster's index does not: -// folderId -> { bytes, newestModified }. Peers ask for it before promoting one of their -// own, so that the node with the data wins rather than the node with the lowest address. -// Null until the first monitor pass, for the same reason promotedFolderIds is: "I hold -// nothing" and "I have not looked" are opposite answers to a peer about to seed. -let folderHoldings = null; +// Where each receive-only folder of an installed app stands in telling peers what this +// node holds of it: folderId -> a state from appMonitoring/folderStates. Peers read it +// before promoting one of their own, so that the node with the data wins rather than +// the node with the lowest address. +const folderStates = new Map(); // This node's place in deciding which holder seeds a folder at a cold start: // folderId -> { stage, bytes, newestModified }. `intent` while this node is elected, @@ -367,8 +366,7 @@ module.exports = { get receiveOnlySyncthingAppsCache() { return receiveOnlySyncthingAppsCache; }, get promotedFolderIds() { return promotedFolderIds; }, set promotedFolderIds(ids) { promotedFolderIds = ids; }, - get folderHoldings() { return folderHoldings; }, - set folderHoldings(map) { folderHoldings = map; }, + get folderStates() { return folderStates; }, get seedMarks() { return seedMarks; }, get syncthingDevicesIDCache() { return syncthingDevicesIDCache; }, get folderHealthCache() { return folderHealthCache; }, diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index e5d7d9e6fe..6c3abf18aa 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -120,6 +120,35 @@ describe('advancedWorkflows tests', () => { expect(globalState.installationInProgress).to.be.false; }); + // A soft registration keeps the app's volumes, so a folder that already has a + // state keeps it; one the spec adds has not been looked at. + it('records a folder the spec adds as not looked at, and keeps the state of one it already has', async () => { + sinon.stub(generalService, 'nodeTier').resolves('cumulus'); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + const insert = sinon.stub(dbHelper, 'insertOneToDatabase').resolves({ insertedId: 'id' }); + sinon.stub(appNetworkLinker, 'checkAppNetworkRequirements').resolves(); + sinon.stub(appInstaller, 'ensureAppDockerNetwork').resolves(); + sinon.stub(appInstaller, 'installApplicationSoft').rejects(new Error('Error pulling image')); + sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + globalState.folderStates.clear(); + globalState.folderStates.set('fluxkept_softapp', { state: 'measured', bytes: 10, newestModified: 1 }); + + await advancedWorkflows.softRegisterAppLocally({ + name: 'softapp', + version: 4, + compose: [ + { name: 'kept', containerData: 'g:/data', cpu: 0.5, ram: 500, hdd: 5 }, + { name: 'added', containerData: 'g:/data', cpu: 0.5, ram: 500, hdd: 5 }, + ], + }, false, makeRes()); + + sinon.assert.called(insert); + expect(globalState.folderStates.get('fluxkept_softapp')).to.deep.equal({ state: 'measured', bytes: 10, newestModified: 1 }); + expect(globalState.folderStates.get('fluxadded_softapp')).to.deep.equal({ state: 'notLookedAt' }); + globalState.folderStates.clear(); + }); + it('does not release a hold it never took', async () => { // Refused because another install owns the node. Lowering the flag here // hands the node to the next caller while that install is still running. @@ -5296,10 +5325,8 @@ describe('advancedWorkflows tests', () => { // What this node publishes about the volume being replaced. Peers rank a seed // on it, so it describes one incarnation and must not outlive it. const armHolding = () => { - volGlobalState.folderHoldings = new Map([ - [identifier, { bytes: 5821604997, newestModified: 200 }], - ['fluxother_TestApp', { bytes: 4096, newestModified: 100 }], - ]); + volGlobalState.folderStates.set(identifier, { state: 'measured', bytes: 5821604997, newestModified: 200 }); + volGlobalState.folderStates.set('fluxother_TestApp', { state: 'measured', bytes: 4096, newestModified: 100 }); volGlobalState.seedMarks.set(identifier, { stage: 'intent', bytes: 5821604997, newestModified: 200 }); volGlobalState.seedMarks.set('fluxother_TestApp', { stage: 'intent', bytes: 4096, newestModified: 100 }); }; @@ -5307,7 +5334,7 @@ describe('advancedWorkflows tests', () => { beforeEach(() => { volGlobalState = require('../../ZelBack/src/services/utils/globalState'); volGlobalState.receiveOnlySyncthingAppsCache.clear(); - volGlobalState.folderHoldings = null; + volGlobalState.folderStates.clear(); volGlobalState.seedMarks.clear(); const hwRequirements = require('../../ZelBack/src/services/appRequirements/hwRequirements'); sinon.stub(hwRequirements, 'getNodeSpecs').resolves({ ssdStorage: 10000 }); @@ -5485,12 +5512,12 @@ describe('advancedWorkflows tests', () => { expect(thrown, 'the pre-flight abort did not fire').to.not.equal(null); expect( - volGlobalState.folderHoldings.has(identifier), + volGlobalState.folderStates.get(identifier), 'an aborted pre-flight withdrew a claim whose data is intact', - ).to.equal(true); + ).to.include({ state: 'measured' }); }); - it('drops the published holding at the point of no return', async () => { + it('records the volume as not looked at from the point of no return', async () => { // Once the allocation runs the volume is empty. A claim surviving from the // previous incarnation describes data this node no longer has, and peers rank // the seed on it - so it outranks a node that still holds the app and seeds @@ -5510,11 +5537,11 @@ describe('advancedWorkflows tests', () => { expect(thrown, 'the flow never reached the allocation').to.not.equal(null); expect(thrown.message, 'the flow aborted before the allocation').to.equal('fallocate blocked by test'); expect( - volGlobalState.folderHoldings.has(identifier), + volGlobalState.folderStates.get(identifier), 'the point of no return left a claim for a volume that is now empty', - ).to.equal(false); + ).to.deep.equal({ state: 'notLookedAt' }); // Scoped to the volume being replaced, not a clear of every answer the node holds. - expect(volGlobalState.folderHoldings.has('fluxother_TestApp')).to.equal(true); + expect(volGlobalState.folderStates.get('fluxother_TestApp')).to.include({ state: 'measured' }); expect(volGlobalState.seedMarks.has(identifier), 'the point of no return left an offer to seed an empty volume').to.equal(false); expect(volGlobalState.seedMarks.has('fluxother_TestApp')).to.equal(true); }); diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 763b064b5e..e3d15e09ee 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -5,6 +5,7 @@ const { resetGlobalState } = require('./fixtures/globalState'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); // what upnpService.mapUpnpPort answers for a port mapped, and for one the router refused const MAPPED = { @@ -1379,6 +1380,27 @@ describe('appInstaller tests', () => { expect(removeAppLocallyStub.firstCall.args[4], 'broadcast a removal for an app this node never had').to.equal(false); }); + // From its row on, the app is installed here and a peer may list this node as a + // holder: until the monitor has measured the folder, a peer about to seed waits. + it('records the app\'s receive-only folder as not looked at once its row is written', async () => { + folderStateMap.clear(); + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves()); + + await appInstallerWithDb.registerAppLocally({ ...appSpec, name: 'fluxinst', containerData: 'g:/appdata' }, false, { write: sinon.stub(), end: sinon.stub() }); + + expect(folderStateMap.get('fluxinst')).to.deep.equal({ state: 'notLookedAt' }); + folderStateMap.clear(); + }); + + it('records nothing when it gave up before the app reached the table', async () => { + folderStateMap.clear(); + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves(), { failBeforeRow: true }); + + await appInstallerWithDb.registerAppLocally({ ...appSpec, name: 'fluxinst', containerData: 'g:/appdata' }, false, { write: sinon.stub(), end: sinon.stub() }); + + expect(folderStateMap.has('fluxinst')).to.equal(false); + }); + // A test install writes the app's row like any other and throws it away, so the // announcement must not read that row as a claim. The mark outlives the teardown // that deletes the row: released before it, a cycle in between claims the app. diff --git a/tests/unit/appQueryService.test.js b/tests/unit/appQueryService.test.js index 14265bb8ec..af8f07b270 100644 --- a/tests/unit/appQueryService.test.js +++ b/tests/unit/appQueryService.test.js @@ -1024,8 +1024,8 @@ describe('appQueryService tests', () => { }); beforeEach(() => { - globalState.promotedFolderIds = new Set(['fluxa_a']); - globalState.folderHoldings = new Map([['fluxa_a', { bytes: 5821604997, newestModified: 200 }]]); + globalState.promotedFolderIds = new Set(['fluxw_w']); + globalState.folderStates.set('fluxa_a', { state: 'measured', bytes: 5821604997, newestModified: 200 }); messageHelperStub.createDataMessage.returnsArg(0); // no app installed unless a test says otherwise dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('appsDatabase') }); @@ -1033,7 +1033,7 @@ describe('appQueryService tests', () => { }); afterEach(() => { - globalState.folderHoldings = null; + globalState.folderStates.clear(); globalState.seedMarks.clear(); }); @@ -1074,7 +1074,7 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); expect(result.holding, 'holdings went to a caller that proved nothing').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'], notLookedAt: [] }); + expect(result).to.deep.equal({ ready: true, folders: ['fluxw_w'], notLookedAt: [] }); }); }); @@ -1091,7 +1091,7 @@ describe('appQueryService tests', () => { }); expect(result.holding, 'a body dated in the future served as a credential').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'], notLookedAt: [] }); + expect(result).to.deep.equal({ ready: true, folders: ['fluxw_w'], notLookedAt: [] }); }); // The bound is the fleet's clocks disagreeing, which is honest and must still work. @@ -1150,7 +1150,7 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body: signed({ target: '10.0.0.8:16127' }) }); expect(result.holding, 'a signature captured in flight worked on another node').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'], notLookedAt: [] }); + expect(result).to.deep.equal({ ready: true, folders: ['fluxw_w'], notLookedAt: [] }); }); it('withholds the holdings when this node does not know its own address', async () => { @@ -1192,50 +1192,52 @@ describe('appQueryService tests', () => { // An install its monitor has not reached: a peer about to seed waits on it rather // than read this node as holding nothing. - describe('the folders it holds and has not yet measured', () => { - const synced = (name, mode) => ({ name, version: 8, compose: [{ name, containerData: `${mode}:/data` }] }); + describe('the folders it has not yet looked at', () => { + beforeEach(() => { + fluxNetworkHelperStub.verifySignedFluxnodeMessage.resolves(true); + }); - it('lists a synced folder of an installed app that it has neither measured nor made writable, to any caller', async () => { - dbHelperStub.findInDatabase.resolves([synced('b', 'r'), synced('c', 'g')]); + it('lists a folder not yet looked at, to any caller, and claims nothing of it', async () => { + globalState.folderStates.set('fluxb_b', { state: 'notLookedAt' }); - const result = await appQueryService.promotedFolderHoldings({ body: {} }); + const open = await appQueryService.promotedFolderHoldings({ body: {} }); + const signedAnswer = await appQueryService.promotedFolderHoldings({ body: signed() }); - expect(result.ready).to.equal(true); - expect(result.notLookedAt).to.have.members(['fluxb_b', 'fluxc_c']); + expect(open.notLookedAt).to.deep.equal(['fluxb_b']); + expect(signedAnswer.holding).to.not.have.property('fluxb_b'); }); - it('leaves out a folder it has measured, one it could not read, one it holds writable, and one that does not sync', async () => { - globalState.promotedFolderIds = new Set(['fluxa_a', 'fluxw_w']); - globalState.folderHoldings = new Map([ - ['fluxm_m', { bytes: 10, newestModified: 1 }], - ['fluxu_u', { unreadable: true }], - ]); - dbHelperStub.findInDatabase.resolves([ - synced('m', 'r'), synced('u', 'r'), synced('w', 's'), - { name: 'p', version: 8, compose: [{ name: 'p', containerData: '/data' }] }, - ]); + it('answers no claim, not "not looked at", for a folder held out, one it could not read, and one it has sent', async () => { + globalState.folderStates.set('fluxh_h', { state: 'heldOut' }); + globalState.folderStates.set('fluxn_n', { state: 'noClaim' }); - const result = await appQueryService.promotedFolderHoldings({ body: {} }); + const result = await appQueryService.promotedFolderHoldings({ body: signed() }); expect(result.notLookedAt).to.deep.equal([]); + expect(result.holding).to.deep.equal({ + fluxa_a: { bytes: 5821604997, newestModified: 200 }, + fluxh_h: { unreadable: true }, + fluxn_n: { unreadable: true }, + }); }); - it('lists every synced folder before its monitor has measured any', async () => { - globalState.folderHoldings = null; - dbHelperStub.findInDatabase.resolves([synced('b', 'r')]); + it('leaves a writable folder out of both, whatever its state', async () => { + globalState.folderStates.set('fluxw_w', { state: 'notLookedAt' }); - const result = await appQueryService.promotedFolderHoldings({ body: {} }); + const result = await appQueryService.promotedFolderHoldings({ body: signed() }); - expect(result.notLookedAt).to.deep.equal(['fluxb_b']); + expect(result.notLookedAt).to.deep.equal([]); + expect(result.holding).to.not.have.property('fluxw_w'); }); - it('answers not ready when it cannot read its installed apps', async () => { - dbHelperStub.findInDatabase.rejects(new Error('operation exceeded time limit')); + // The route is open: the answer is what the monitor already recorded, read + // from memory, so a caller cannot make this node read its database. + it('reads no database to answer', async () => { + globalState.folderStates.set('fluxb_b', { state: 'notLookedAt' }); - const result = await appQueryService.promotedFolderHoldings({ body: {} }); + await appQueryService.promotedFolderHoldings({ body: {} }); - expect(result.ready, 'a node that cannot say which folders it has measured answered ready').to.equal(false); - expect(result.notLookedAt).to.deep.equal([]); + sinon.assert.notCalled(dbHelperStub.findInDatabase); }); }); diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 44cffa8321..959bdc2af2 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -4,6 +4,7 @@ const proxyquire = require('proxyquire').noCallThru(); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutcome'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); describe('appUninstaller tests', () => { describe('cleanupPorts tests', () => { @@ -619,7 +620,6 @@ describe('appUninstaller tests', () => { installationInProgress: false, runningAppsCache: new Set(), receiveOnlySyncthingAppsCache: new Map(), - folderHoldings: new Map(), seedMarks: new Map(), // the two operations this suite sets, named as the module names them operationHolding() { @@ -967,21 +967,19 @@ describe('appUninstaller tests', () => { const uninstaller = buildUninstaller(composedSpec); // Keyed by the app IDENTIFIER, which is what the folder id and the cleanup // both use - not the component name the removal is asked for. - globalStateStub.folderHoldings = new Map([ - ['fluxcomp1_testapp', { bytes: 5821604997, newestModified: 200 }], - ['otherapp', { bytes: 4096, newestModified: 100 }], - ]); + folderStateMap.set('fluxcomp1_testapp', { state: 'measured', bytes: 5821604997, newestModified: 200 }); + folderStateMap.set('otherapp', { state: 'measured', bytes: 4096, newestModified: 100 }); globalStateStub.seedMarks.set('fluxcomp1_testapp', { stage: 'intent', bytes: 5821604997, newestModified: 200 }); globalStateStub.seedMarks.set('otherapp', { stage: 'intent', bytes: 4096, newestModified: 100 }); await uninstaller.removeAppLocally('comp1_testapp', res, true); - expect(globalStateStub.folderHoldings.has('fluxcomp1_testapp'), 'a removed app still claims to hold its data').to.equal(false); + expect(folderStateMap.has('fluxcomp1_testapp'), 'a removed app still claims to hold its data').to.equal(false); expect(globalStateStub.seedMarks.has('fluxcomp1_testapp'), 'a removed app still offers to seed').to.equal(false); expect(globalStateStub.seedMarks.has('otherapp')).to.equal(true); // Scoped to what was removed - a component-scoped removal is not a clear of // everything this node has answered for. - expect(globalStateStub.folderHoldings.has('otherapp')).to.equal(true); + expect(folderStateMap.has('otherapp')).to.equal(true); }); it('pairs the seam with the durable runtime-state clear (same identifiers)', async () => { diff --git a/tests/unit/folderStates.test.js b/tests/unit/folderStates.test.js new file mode 100644 index 0000000000..db0ffee0e8 --- /dev/null +++ b/tests/unit/folderStates.test.js @@ -0,0 +1,120 @@ +const { expect } = require('chai'); +const folderStates = require('../../ZelBack/src/services/appMonitoring/folderStates'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); + +describe('folderStates tests', () => { + beforeEach(() => { + folderStateMap.clear(); + }); + + afterEach(() => { + folderStateMap.clear(); + }); + + describe('installed', () => { + it('records each g: and r: component of an app, and no s: or unsynced one', () => { + folderStates.installed({ + name: 'app', + version: 4, + compose: [ + { name: 'g', containerData: 'g:/data' }, + { name: 'r', containerData: 'r:/data' }, + { name: 's', containerData: 's:/data' }, + { name: 'plain', containerData: '/data' }, + ], + }); + + expect([...folderStateMap.keys()]).to.have.members(['fluxg_app', 'fluxr_app']); + expect(folderStateMap.get('fluxg_app')).to.deep.equal({ state: 'notLookedAt' }); + }); + + it('records the single folder of an app from before compose', () => { + folderStates.installed({ name: 'legacy', version: 3, containerData: 'g:/data' }); + + expect(folderStateMap.get('fluxlegacy')).to.deep.equal({ state: 'notLookedAt' }); + }); + + it('keeps the state of a folder it already has', () => { + folderStateMap.set('fluxlegacy', { state: 'heldOut' }); + + folderStates.installed({ name: 'legacy', version: 3, containerData: 'g:/data' }); + + expect(folderStateMap.get('fluxlegacy')).to.deep.equal({ state: 'heldOut' }); + }); + }); + + describe('measured', () => { + it('records what it holds as a claim', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + + expect(folderStates.claimOf('fluxa')).to.deep.equal({ bytes: 10, newestModified: 2 }); + }); + + it('records holdings it could not read as no claim', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + folderStates.measured('fluxa', null); + + expect(folderStateMap.get('fluxa')).to.deep.equal({ state: 'noClaim' }); + expect(folderStates.claimOf('fluxa')).to.equal(folderStates.NO_CLAIM_ANSWER); + }); + }); + + describe('claimOf', () => { + it('claims nothing, and is not no claim, for a folder not looked at or not held', () => { + folderStateMap.set('fluxa', { state: 'notLookedAt' }); + + expect(folderStates.claimOf('fluxa')).to.equal(undefined); + expect(folderStates.claimOf('fluxmissing')).to.equal(undefined); + }); + + it('answers no claim for a folder held out', () => { + folderStates.heldOut('fluxa'); + + expect(folderStates.claimOf('fluxa')).to.equal(folderStates.NO_CLAIM_ANSWER); + }); + }); + + describe('the folder lifecycle', () => { + it('makes a promoted folder no claim, and keeps it so once it no longer sends', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + + folderStates.sending('fluxa'); + + expect(folderStates.answer(new Set())).to.deep.equal({ holding: { fluxa: { unreadable: true } }, notLookedAt: [] }); + }); + + it('makes a recreated volume not looked at', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + + folderStates.recreated('fluxa'); + + expect(folderStates.answer(new Set()).notLookedAt).to.deep.equal(['fluxa']); + }); + + it('forgets a removed folder', () => { + folderStates.heldOut('fluxa'); + + folderStates.removed('fluxa'); + + expect(folderStateMap.has('fluxa')).to.equal(false); + }); + }); + + describe('answer', () => { + it('lists folders not looked at apart from what it holds, and leaves out writable ones', () => { + folderStateMap.set('fluxwait', { state: 'notLookedAt' }); + folderStateMap.set('fluxheld', { state: 'measured', bytes: 5, newestModified: 1 }); + folderStateMap.set('fluxout', { state: 'heldOut' }); + folderStateMap.set('fluxsend', { state: 'measured', bytes: 9, newestModified: 9 }); + folderStateMap.set('fluxsendwait', { state: 'notLookedAt' }); + + const { holding, notLookedAt } = folderStates.answer(new Set(['fluxsend', 'fluxsendwait'])); + + expect(notLookedAt).to.deep.equal(['fluxwait']); + expect(holding).to.deep.equal({ + fluxheld: { bytes: 5, newestModified: 1 }, + fluxout: { unreadable: true }, + }); + }); + }); +}); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 111da98bd4..8e93feb9b7 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -193,6 +193,7 @@ const stateMachine = proxyquire('../../ZelBack/src/services/appMonitoring/syncth describe('syncthingFolderStateMachine tests', () => { beforeEach(() => { + globalStateMock.folderStates.clear(); peerIdentityMock.reset(); primaryRoleMock.holdAsStandby.resetHistory(); peerComponentMock.componentStateOnPeers.reset(); @@ -903,6 +904,26 @@ describe('syncthingFolderStateMachine tests', () => { sinon.assert.notCalled(appReconcilerMock.setControllerDesired); }); + // A ready standby is not measured, so a peer about to seed must not wait on it: + // every state but a measurement it already holds becomes no claim. + [ + ['not looked at', { state: 'notLookedAt' }, { state: 'noClaim' }], + ['held out', { state: 'heldOut' }, { state: 'noClaim' }], + ['measured', { state: 'measured', bytes: 7, newestModified: 3 }, { state: 'measured', bytes: 7, newestModified: 3 }], + ].forEach(([what, before, after]) => { + it(`records a ready single-writer standby it found ${what} as ${after.state}`, async () => { + mockParams.containerDataFlags = 'g'; + mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); + dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); + globalStateMock.folderStates.set('test-app', before); + + await stateMachine.manageFolderSyncState(mockParams); + + expect(globalStateMock.folderStates.get('test-app')).to.deep.equal(after); + }); + }); + // A receiveonly folder keeps its own deviations, owner and mode included, and // the primary counts every one of them as outstanding for this node. it('reverts a ready single-writer standby whose directories differ from the cluster', async () => { @@ -1158,18 +1179,16 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); - expect(globalStateMock.folderHoldings.get('test-app').bytes).to.equal(5821604997); + expect(globalStateMock.folderStates.get('test-app')).to.include({ state: 'measured', bytes: 5821604997 }); }); // A read it cannot make is not a smaller claim, it is no claim - and the node // stands down on exactly that. Left up, the last good figure makes every peer // defer to a node that is refusing to lead, and nobody seeds. Recorded as - // unreadable rather than removed, so the folder is not one it has yet to look at. - it('records the folder unreadable, claiming nothing, when it can no longer read what it holds', async () => { - globalStateMock.folderHoldings = new Map([ - ['test-app', { bytes: 5821604997, newestModified: 200 }], - ['other-app', { bytes: 900, newestModified: 100 }], - ]); + // no claim rather than removed, so the folder is not one it has yet to look at. + it('records the folder as no claim when it can no longer read what it holds', async () => { + globalStateMock.folderStates.set('test-app', { state: 'measured', bytes: 5821604997, newestModified: 200 }); + globalStateMock.folderStates.set('other-app', { state: 'measured', bytes: 900, newestModified: 100 }); fsMock.promises.readdir.resolves([]); mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1, leaderStreak: 5, @@ -1182,8 +1201,8 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); - expect(globalStateMock.folderHoldings.get('test-app'), 'an unreadable folder still claims its last good figure').to.deep.equal({ unreadable: true }); - expect(globalStateMock.folderHoldings.has('other-app'), 'another folder\'s claim is not this folder\'s to withdraw').to.equal(true); + expect(globalStateMock.folderStates.get('test-app'), 'an unreadable folder still claims its last good figure').to.deep.equal({ state: 'noClaim' }); + expect(globalStateMock.folderStates.get('other-app'), 'another folder\'s claim is not this folder\'s to withdraw').to.include({ state: 'measured' }); }); // The status read for the decision is the one the disk is judged against. Read @@ -2460,7 +2479,7 @@ describe('syncthingFolderStateMachine tests', () => { mockParams.electedSiblings = ['sibling-app']; // no node writes the app's other folder yet, so the election decides this one appQueryServiceMock.holdsComponent.resolves(false); - globalStateMock.folderHoldings = new Map([['sibling-app', { bytes: 0, newestModified: 0 }]]); + globalStateMock.folderStates.set('sibling-app', { state: 'measured', bytes: 0, newestModified: 0 }); const result = await stateMachine.manageFolderSyncState(mockParams); @@ -2594,7 +2613,7 @@ describe('syncthingFolderStateMachine tests', () => { const seeds = (result) => result.cache.designationPending === true; afterEach(() => { - globalStateMock.folderHoldings?.delete(SIBLING); + globalStateMock.folderStates.clear(); }); describe('a component the spec has just added', () => { @@ -2689,8 +2708,7 @@ describe('syncthingFolderStateMachine tests', () => { }); appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [localEntry('a.dat', 100)] })); - globalStateMock.folderHoldings = globalStateMock.folderHoldings || new Map(); - globalStateMock.folderHoldings.set(SIBLING, { bytes: 0, newestModified: 0 }); + globalStateMock.folderStates.set(SIBLING, { state: 'measured', bytes: 0, newestModified: 0 }); }; it('ranks the whole app, so the node holding the most of it seeds every folder', async () => { @@ -2709,7 +2727,7 @@ describe('syncthingFolderStateMachine tests', () => { it('falls back to the address order while this node cannot say what it holds of the other folder', async () => { split(); - globalStateMock.folderHoldings.delete(SIBLING); + globalStateMock.folderStates.delete(SIBLING); expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); }); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index f725038368..b55670700d 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -5,6 +5,8 @@ const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); const eventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); +const folderStates = require('../../ZelBack/src/services/appMonitoring/folderStates'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); // Create mocks for all dependencies const dbHelperMock = { @@ -1438,6 +1440,93 @@ describe('syncthingMonitor tests', () => { }); }); + // What a peer about to seed reads of each receive-only folder here: it waits on one + // this node has not looked at, so every folder a pass reaches must leave that state. + describe('the folders it tells peers it has not looked at', () => { + // Folder ids already in identifier form, so the state the install records and the + // one the pass records name the same folder. + const installedApps = (apps) => mockInstalledAppsFn.resolves({ status: 'success', data: apps }); + const notLookedAt = () => folderStates.answer(new Set()).notLookedAt; + + beforeEach(() => { + folderStateMap.clear(); + }); + + afterEach(() => { + folderStateMap.clear(); + }); + + // After a restart the volumes still hold their data and nothing has been measured, + // so a peer must wait on every folder until this node has looked. + it('records every receive-only folder of an installed app as not looked at, and no other', async () => { + installedApps([ + { name: 'fluxg', version: 3, containerData: 'g:/appdata' }, + { name: 'fluxr', version: 3, containerData: 'r:/appdata' }, + { name: 'fluxs', version: 3, containerData: 's:/appdata' }, + { name: 'fluxplain', version: 3, containerData: '/appdata' }, + ]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(notLookedAt()).to.have.members(['fluxg', 'fluxr']); + }); + + it('keeps a state a folder already has', async () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + folderStateMap.set('fluxg', { state: 'measured', bytes: 10, newestModified: 1 }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'measured', bytes: 10, newestModified: 1 }); + }); + + // A volume that did not mount is never measured. Left not looked at, every peer + // would wait on this node for as long as the mount stays broken, and the app would + // start nowhere. + it('records a folder held out because its volume did not mount as held out, which no peer waits on', async () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingEventsConsumerMock.mountVerifyPendingIds.returns(['fluxg']); + syncthingFolderStateMachineMock.verifyFolderMountSafety.resolves({ isSafe: false, isMounted: false, reason: 'empty_unmounted_directory' }); + volumeServiceMock.ensureAppVolumeMounted.resolves({ mounted: false, reason: 'volume_file_missing' }); + syncthingServiceMock.getConfigFolders.resolves([{ id: 'fluxg', path: '/apps/fluxg', type: 'receiveonly' }]); + syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); + expect(notLookedAt()).to.deep.equal([]); + }); + + it('records a folder whose volume directory is not mounted as held out', async () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingMonitorHelpersMock.ensureStfolderExists.resolves(false); + syncthingServiceMock.getConfigFolders.resolves([]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); + }); + + it('records a folder whose locations could not be read as held out', async () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingMonitorHelpersMock.ensureStfolderExists.resolves(true); + dbHelperMock.findInDatabase.rejects(new Error('database unavailable')); + syncthingServiceMock.getConfigFolders.resolves([]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); + }); + }); + describe('holds a busy app out of the config write', () => { const syncingApp = { name: 'testapp', version: 3, containerData: 'g:/appdata' }; // Bound while this file loads, which is when the module under test bound its @@ -1495,13 +1584,11 @@ describe('syncthingMonitor tests', () => { // question. Promotion answers it - everything this node holds is published // from here - so the claim stops being true as the write lands, which is // where promotedFolderIds is reconciled for the same reason. - it('drops the published holding of a folder it promotes', async () => { + it('withdraws the published holding of a folder it promotes', async () => { writesAFolder(); syncthingMonitorHelpersMock.getContainerDataFlags.returns('r'); - globalState.folderHoldings = new Map([ - ['testapp', { bytes: 5821604997, newestModified: 200 }], - ['untouchedapp', { bytes: 4096, newestModified: 100 }], - ]); + globalState.folderStates.set('testapp', { state: 'measured', bytes: 5821604997, newestModified: 200 }); + globalState.folderStates.set('untouchedapp', { state: 'measured', bytes: 4096, newestModified: 100 }); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); @@ -1511,10 +1598,10 @@ describe('syncthingMonitor tests', () => { expect(put, 'nothing was written, so this asserts nothing').to.not.equal(undefined); expect(put.args[1].find((f) => f.id === 'testapp')?.type).to.equal('sendreceive'); - expect(globalState.folderHoldings.has('testapp'), 'a promoted folder still claims receive-only holdings').to.equal(false); - // A folder this pass did not promote keeps its claim: the drop is scoped to - // what changed, not a clear of everything the node has answered for. - expect(globalState.folderHoldings.has('untouchedapp')).to.equal(true); + expect(globalState.folderStates.get('testapp'), 'a promoted folder still claims receive-only holdings').to.deep.equal({ state: 'noClaim' }); + // A folder this pass did not promote keeps its claim: the withdrawal is scoped + // to what changed, not a clear of everything the node has answered for. + expect(globalState.folderStates.get('untouchedapp')).to.include({ state: 'measured' }); }); // The election reads a folder as ready to start once it is writable, so a From b9ba7c84cda9086abde3ffddcba99dda078a380e Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:12:17 +0100 Subject: [PATCH 290/369] fix(syncthing): a pass counts each folder it holds out for an unsafe mount Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appMonitoring/syncthingMonitor.js | 1 + 1 file changed, 1 insertion(+) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index f2b04ebb1e..dfb588dc4c 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -786,6 +786,7 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { for (const { appId, reason } of unmountedApps) { unsafeFolderIds.add(appId); folderStates.heldOut(appId); + fluxEventBus.count('syncthing:folderPass', appId, 'mountUnsafe'); // The container is the other writer over the bad mount, and holding it // asks nothing of syncthing: a component that syncs the folder, or one // whose spec cannot be read to say it does not, is held before the From 79a164532c8b2117b9916c389958e05b2a68a1d3 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:12:17 +0100 Subject: [PATCH 291/369] test(harness): 2253 seeds a cold start on the lowest address while the other holder's volume will not mount Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/volume-fixture.js | 16 ++- ...did-not-mount-does-not-hold-up-the-seed.js | 108 ++++++++++++++++++ 2 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js diff --git a/test-infra/runner/framework/volume-fixture.js b/test-infra/runner/framework/volume-fixture.js index c4a0f8a028..57ba0a5876 100644 --- a/test-infra/runner/framework/volume-fixture.js +++ b/test-infra/runner/framework/volume-fixture.js @@ -26,6 +26,8 @@ export const APP_GID = 33; export const appId = (name) => `flux${name}_${name}`; export const volumeRoot = (name) => `/mnt/appdata/flux-apps/${appId(name)}`; export const volumeImage = (name) => `/mnt/appdata/${appId(name)}FLUXFSVOL`; +const folderRoot = (folderId) => `/mnt/appdata/flux-apps/${folderId}`; +const folderImage = (folderId) => `/mnt/appdata/${folderId}FLUXFSVOL`; // Single-quote for the shell. Fixture content is chosen by the suites, but a // literal $ or backtick in one must stay a literal rather than becoming @@ -43,12 +45,20 @@ async function run(container, command, what) { } /** - * Takes an app's volume away for good: unmounted, its directory no longer + * Takes a component's volume away for good: unmounted, its directory no longer * immutable, and its image removed, so nothing on the node can mount it again. + * @param {string} folderId The component's identifier, flux_. + */ +export async function destroyFolderVolume(container, folderId) { + const root = folderRoot(folderId); + await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)} && rm -f ${sq(folderImage(folderId))}`, `destroy the volume of ${folderId}`); +} + +/** + * destroyFolderVolume for an app whose one component shares its name. */ export async function destroyVolume(container, name) { - const root = volumeRoot(name); - await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)} && rm -f ${sq(volumeImage(name))}`, `destroy the volume of ${name}`); + await destroyFolderVolume(container, appId(name)); } /** diff --git a/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js b/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js new file mode 100644 index 0000000000..bec3e996d6 --- /dev/null +++ b/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js @@ -0,0 +1,108 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { syncthingSeedIndex } from '../framework/g-app-placement.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { destroyFolderVolume } from '../framework/volume-fixture.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A cold start of a g: app where one holder comes back from a restart with the +// app's volume gone: the image is removed, so the mount check cannot repair it and +// holds the folder out of every pass. That holder never measures the folder. +// +// The other holder, the lowest address, decides the seed. It waits on a holder +// that has not yet looked at the folder, so a held-out folder must read as looked +// at with no claim: read as not looked at, it holds the seed up for as long as the +// volume stays gone, and the app starts nowhere. +// +// The lowest address is held before it measures until the broken holder has held +// the folder out, so its decision is made against that answer. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; + +describe('a holder whose volume did not mount does not hold up the seed', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eheldout${Date.now()}`; + const COMPONENT = 'data'; + const HOLDERS = [0, 1]; + const LOWEST = syncthingSeedIndex(HOLDERS); + const BROKEN = HOLDERS.find((i) => i !== LOWEST); + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: [COMPONENT], holders: HOLDERS, + }); + const folder = folderOf(COMPONENT); + const seeded = (i) => client(i).getDecisionCount('syncthing:seedDecision', folder, 'seeded'); + const waitedOnLook = (i) => client(i).getDecisionCount('syncthing:seedDecision', folder, 'holderNotLookedAt'); + const folderPasses = (i) => client(i).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + const heldOutPasses = (i) => client(i).getDecisionCount('syncthing:folderPass', folder, 'mountUnsafe'); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + + await pushTestApp(appName, 'v1', 'heldout', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart(HOLDERS, folder); + await client(LOWEST).holdCheckpoint(BEFORE_HOLDINGS, folder); + const from = client(LOWEST).getLastEventId(); + await installOnNodes(env, app, HOLDERS); + await client(LOWEST).waitForEvent( + 'checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, + 300000, + { afterId: from }, + ); + }); + + after(async function () { + this.timeout(60000); + await client(LOWEST)?.releaseAllCheckpoints().catch(() => {}); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + it('seeds on the lowest address while the other holder holds the folder out', async function () { + this.timeout(600000); + expect(subnet.nodeIp(LOWEST + 1) < subnet.nodeIp(BROKEN + 1), 'fixture: the broken holder is not the lowest address').to.equal(true); + + await crashFluxos(client(BROKEN).container, { hold: true }); + await destroyFolderVolume(client(BROKEN).container, folder); + await releaseFluxos(client(BROKEN).container); + await waitFor(async () => await heldOutPasses(BROKEN) >= 1, { + timeout: 300000, interval: 2000, label: 'the restarted holder holds the folder out', + }); + expect(await seeded(LOWEST), 'fixture: the lowest address seeded before it was released').to.equal(0); + + const from = await folderPasses(LOWEST); + await client(LOWEST).releaseCheckpoint(BEFORE_HOLDINGS, folder); + await waitHolding(async () => { + await oneHolderAtMost(); + return await seeded(LOWEST) >= 1 || await folderPasses(LOWEST) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label: `the lowest address seeds, or ${SETTLE_PASSES} folder passes go by without it` }); + + expect(await seeded(LOWEST), `seeds on the lowest address in ${SETTLE_PASSES} passes (passes it waited on a holder that had not looked: ${await waitedOnLook(LOWEST)})`).to.be.at.least(1); + expect(await waitedOnLook(LOWEST), 'passes the lowest address waited on the held-out holder').to.equal(0); + expect(await seeded(BROKEN), 'the broken holder seeded').to.equal(0); + expect(await heldOutPasses(BROKEN), 'fixture: the broken holder still holds the folder out').to.be.at.least(1); + }); +}); From 242be0ab25d0e576279153153731e7f47677e783 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:23:10 +0100 Subject: [PATCH 292/369] test(harness): 2253 counts the waits of a pass that opens after the holder has held the folder out Co-Authored-By: Claude Opus 5.5 --- ...ume-did-not-mount-does-not-hold-up-the-seed.js | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js b/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js index bec3e996d6..4997f50275 100644 --- a/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js +++ b/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js @@ -24,11 +24,13 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // volume stays gone, and the app starts nowhere. // // The lowest address is held before it measures until the broken holder has held -// the folder out, so its decision is made against that answer. +// the folder out. The held pass read its peers before that, so the decision under +// test is the one a later pass makes, held where it opens until the first is done. const subnet = getSubnetConfig(); const APP_UID = 1000; const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const AFTER_FOLDER_READ = 'syncthing:afterFolderRead'; const SETTLE_PASSES = 6; describe('a holder whose volume did not mount does not hold up the seed', function () { @@ -93,15 +95,20 @@ describe('a holder whose volume did not mount does not hold up the seed', functi }); expect(await seeded(LOWEST), 'fixture: the lowest address seeded before it was released').to.equal(0); - const from = await folderPasses(LOWEST); + await client(LOWEST).holdCheckpoint(AFTER_FOLDER_READ); + const opened = client(LOWEST).getLastEventId(); await client(LOWEST).releaseCheckpoint(BEFORE_HOLDINGS, folder); + await client(LOWEST).waitForEvent('checkpoint:held', (d) => d.name === AFTER_FOLDER_READ, 300000, { afterId: opened }); + const waitsBefore = await waitedOnLook(LOWEST); + const from = await folderPasses(LOWEST); + await client(LOWEST).releaseCheckpoint(AFTER_FOLDER_READ); await waitHolding(async () => { await oneHolderAtMost(); return await seeded(LOWEST) >= 1 || await folderPasses(LOWEST) >= from + SETTLE_PASSES; }, { timeout: 300000, interval: 2000, label: `the lowest address seeds, or ${SETTLE_PASSES} folder passes go by without it` }); - expect(await seeded(LOWEST), `seeds on the lowest address in ${SETTLE_PASSES} passes (passes it waited on a holder that had not looked: ${await waitedOnLook(LOWEST)})`).to.be.at.least(1); - expect(await waitedOnLook(LOWEST), 'passes the lowest address waited on the held-out holder').to.equal(0); + expect(await seeded(LOWEST), `seeds on the lowest address in ${SETTLE_PASSES} passes (passes it waited on a holder that had not looked: ${await waitedOnLook(LOWEST) - waitsBefore})`).to.be.at.least(1); + expect(await waitedOnLook(LOWEST) - waitsBefore, 'passes the lowest address waited on the held-out holder').to.equal(0); expect(await seeded(BROKEN), 'the broken holder seeded').to.equal(0); expect(await heldOutPasses(BROKEN), 'fixture: the broken holder still holds the folder out').to.be.at.least(1); }); From 5d2fec32400f309ff585bbb71696cac5602ef6e8 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 20:36:16 +0100 Subject: [PATCH 293/369] fix(syncthing): a local change that is the only copy is found from the folder's list of changes, and the one found is asked about first while the folder keeps its changes syncthing marks the record that is the cluster's version, so each entry of db/localchanged says by itself whether no other device holds it: the check reads the pages it pages through and asks nothing per item. While a folder keeps its changes, the item that answered is asked about first on the next pass, and the list is read again only when it no longer answers. Co-Authored-By: Claude Opus 5.5 --- .../syncthingFolderStateMachine.js | 65 +++++++++----- ...-went-quiet-and-then-died-is-taken-over.js | 5 ++ .../unit/syncthingFolderStateMachine.test.js | 84 +++++++++++++++---- 3 files changed, 119 insertions(+), 35 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index fae46e5345..8475be5d2d 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1291,39 +1291,62 @@ async function nudgeFolderDevices(folderId) { // syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. const RECEIVE_ONLY_CHANGE_FLAG = 8; +// syncthing's FlagLocalGlobal: this device's record of the item is the cluster's version. +const GLOBAL_VERSION_FLAG = 16; const LOCAL_CHANGE_PAGE_SIZE = 1000; /** - * Whether a receive-only folder holds a local change that is the cluster's data and - * that no other device holds: an item whose cluster version is this node's own change - * (db/file, global localFlags) and whose history names another device (a version - * vector counter that is not this node's). syncthing reverts such an item by deleting + * Whether this node's record of an item is a local change to data from another device + * that no other device holds. A receive-only change is the cluster's version only when + * no other device holds a valid version of the item, and its history names another + * device when the data came from elsewhere. syncthing reverts such an item by deleting * it - the only copy of data the app wrote elsewhere - and a directory that is not * empty cannot be deleted, so that revert fails on every pass. * * An item whose history is this node's alone was made here, never by the app's * writer: reverting deletes it, as it should. + * @param {object} item This node's record of the item, as db/localchanged and db/file's + * `local` give it. + * @param {string} ownCounter This node's version-vector prefix, `:`. + * @returns {boolean} + */ +function isOnlyCopyHere(item, ownCounter) { + const flags = item?.localFlags ?? 0; + // eslint-disable-next-line no-bitwise + const changedHere = Boolean(flags & RECEIVE_ONLY_CHANGE_FLAG); + // eslint-disable-next-line no-bitwise + const clusterVersion = Boolean(flags & GLOBAL_VERSION_FLAG); + const fromElsewhere = (item?.version ?? []).some((counter) => !counter.startsWith(ownCounter)); + return changedHere && clusterVersion && fromElsewhere; +} + +/** + * A local change in a receive-only folder that is the only copy of data from another + * device (isOnlyCopyHere), read from the folder's own list of local changes. + * + * `known` is the item that answered last time. It is asked about first, so a folder + * keeping its changes costs one request a pass while that item still answers; the + * list is read only when it does not. * @param {string} appId - The Syncthing folder ID - * @returns {Promise} null when the changes could not be read + * @param {string} [known] - The item that answered last time + * @returns {Promise} the item's name; false when there is none; null + * when the changes could not be read */ -async function localChangeHeldOnlyHere(appId) { +async function localChangeHeldOnlyHere(appId, known) { try { const deviceId = await syncthingService.getDeviceId(); if (!deviceId) return null; const ownCounter = `${deviceId.substring(0, 7)}:`; + if (known) { + const answer = await syncthingService.getDbFile(appId, known).catch(() => null); + if (isOnlyCopyHere(answer?.local, ownCounter)) return known; + } for (let page = 1; ; page += 1) { // eslint-disable-next-line no-await-in-loop const batch = (await syncthingService.getDbLocalChanged(appId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; if (!Array.isArray(batch)) return page > 1 ? false : null; - // eslint-disable-next-line no-restricted-syntax - for (const item of batch) { - // eslint-disable-next-line no-await-in-loop - const { global, local } = await syncthingService.getDbFile(appId, item.name); - // eslint-disable-next-line no-bitwise - const onlyHere = Boolean((global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG); - const fromElsewhere = (local?.version ?? []).some((counter) => !counter.startsWith(ownCounter)); - if (onlyHere && fromElsewhere) return true; - } + const found = batch.find((item) => isOnlyCopyHere(item, ownCounter)); + if (found) return found.name; if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return false; } } catch (error) { @@ -1348,8 +1371,9 @@ async function localChangeHeldOnlyHere(appId) { * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status - * @param {Object} [folderCache] - The folder's cache entry: `keepingChanges` marks a folder - * already said to be keeping its changes, so that is said once, when it starts + * @param {Object} [folderCache] - The folder's cache entry: `keepingChanges` names the + * item that is the only copy while the folder keeps its changes, so keeping them is said + * once, when it starts, and the item is asked about first on the next pass * @returns {Promise} True when the folder has local changes it does not * keep: a revert was asked for, or which changes other devices hold could not be read */ @@ -1363,14 +1387,15 @@ async function revertLocalChangesIfSynced(appId, syncStatus, folderCache) { } const files = syncStatus.receiveOnlyChangedFiles || 0; const directories = syncStatus.receiveOnlyChangedDirectories || 0; - const heldOnlyHere = await localChangeHeldOnlyHere(appId); + const heldOnlyHere = await localChangeHeldOnlyHere(appId, cache.keepingChanges); if (heldOnlyHere === null) { log.warn(`revertLocalChangesIfSynced - ${appId}: could not read which of its ${changed} local change(s) other devices hold; neither reverting nor promoting this pass`); return true; } if (heldOnlyHere) { - if (cache.keepingChanges) return false; - cache.keepingChanges = true; + const keeping = Boolean(cache.keepingChanges); + cache.keepingChanges = heldOnlyHere; + if (keeping) return false; log.warn(`revertLocalChangesIfSynced - ${appId} holds a local change to data from another device that no other device holds (${files} files, ${directories} directories changed); keeping its changes, a revert would delete it`); fluxEventBus.publish('syncthing:localChangesKept', { folder: appId, files, directories }); return false; diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index ecd92d9d18..055805ef60 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -42,6 +42,8 @@ const MONITOR_PASSES_MS = 30000; const TAKEOVER_MS = 10 * 60 * 1000; // syncthing's FlagLocalReceiveOnly const RECEIVE_ONLY_CHANGE_FLAG = 8; +// syncthing's FlagLocalGlobal +const GLOBAL_VERSION_FLAG = 16; const SAVE = 'appdata/save-b'; const SAVE_CONTENT = 'written by the primary'; @@ -119,6 +121,9 @@ describe('a primary that went quiet and then died is taken over', function () { }, { timeout: 60000, interval: 2000, label: 'the file and the directory are changes no other device holds' }); expect(record.local.version.some((counter) => counter.startsWith(`${primaryDevice.substring(0, 7)}:`)), 'the changed file carries no history from the primary, so it is not the primary\'s data').to.equal(true); + // eslint-disable-next-line no-bitwise + expect((record.local.localFlags & GLOBAL_VERSION_FLAG) !== 0, + 'the standby\'s own record of the change is not marked the cluster\'s version, so its list of local changes cannot say no other device holds it').to.equal(true); expect(heldOnlyHere(await getFileInfo(client(A), folder, 'appdata/seed-data')), 'the unchanged file is held only here too, so the standby\'s copy no longer reads as synced').to.equal(false); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 8e93feb9b7..96571e850c 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2332,8 +2332,12 @@ describe('syncthingFolderStateMachine tests', () => { // A change whose cluster version is this node's own is held by no other device: // syncthing reverts it by deleting it. One to data another device wrote is the // only copy of that data, so it is kept and promoted with the rest; one this node - // made itself is reverted, as any pollution is. + // made itself is reverted, as any pollution is. Each is decided from the folder's + // list of local changes: syncthing marks the entry that is the cluster's version. describe('a local change no other device holds', () => { + const CHANGED = 8; + const CHANGED_AND_GLOBAL = 8 | 16; + const changes = (...files) => syncthingServiceMock.getDbLocalChanged.resolves({ files }); const pollutedCandidate = () => { mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1 }); mockParams.appLocation.resolves([ @@ -2344,20 +2348,30 @@ describe('syncthingFolderStateMachine tests', () => { globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, }); }; + const readyStandby = () => { + mockParams.containerDataFlags = 'g'; + mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); + dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, + }); + }; - it('is kept, and the folder is promoted', async () => { + it('is kept, and the folder is promoted, decided from the list alone', async () => { pollutedCandidate(); - syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { localFlags: 8, version: ['PRIMARY:5', 'LOCAL-D:9'] } }); + changes({ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }); const result = await stateMachine.manageFolderSyncState(mockParams); sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.notCalled(syncthingServiceMock.getDbFile); expect(result.cache.restarted, 'a change only this node holds kept the folder from promoting').to.be.true; }); it('is reverted when this node made it', async () => { pollutedCandidate(); - syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { localFlags: 8, version: ['LOCAL-D:9'] } }); + changes({ name: 'appdata/made-here', localFlags: CHANGED_AND_GLOBAL, version: ['LOCAL-D:9'] }); const result = await stateMachine.manageFolderSyncState(mockParams); @@ -2365,25 +2379,46 @@ describe('syncthingFolderStateMachine tests', () => { expect(result.cache.restarted).to.be.false; }); + it('is reverted when another device holds the cluster\'s version', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', localFlags: CHANGED, version: ['PRIMARY:5', 'LOCAL-D:9'] }); + + await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + it('is found behind changes other devices hold', async () => { pollutedCandidate(); - syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'appdata/held' }, { name: 'appdata/only-here' }] }); - syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ global: { localFlags: 8 }, local: { version: ['PRIMARY:5'] } }); + changes( + { name: 'appdata/held', localFlags: CHANGED, version: ['PRIMARY:5'] }, + { name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5'] }, + ); await stateMachine.manageFolderSyncState(mockParams); sinon.assert.notCalled(syncthingServiceMock.dbRevert); }); - it('is kept on a ready standby', async () => { - mockParams.containerDataFlags = 'g'; - mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; - mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); - dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); - syncthingServiceMock.getDbStatus.resolves({ - globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, + it('is found on a later page', async () => { + pollutedCandidate(); + const held = Array.from({ length: 1000 }, (_, k) => ({ name: `appdata/held-${k}`, localFlags: CHANGED, version: ['PRIMARY:5'] })); + syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 1).resolves({ files: held }); + syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 2).resolves({ + files: [{ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5'] }], + }); + + await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('is kept on a ready standby, said once, and asked about first on the next pass', async () => { + readyStandby(); + changes({ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }); + syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ + local: { localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }, }); - syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 8 }, local: { version: ['PRIMARY:5', 'LOCAL-D:9'] } }); const publish = sinon.stub(fluxEventBus, 'publish'); try { await stateMachine.manageFolderSyncState(mockParams); @@ -2395,14 +2430,33 @@ describe('syncthingFolderStateMachine tests', () => { 'syncthing:localChangesKept', { folder: 'test-app', files: 0, directories: 1 }, ); + sinon.assert.calledOnce(syncthingServiceMock.getDbLocalChanged); + sinon.assert.calledOnceWithExactly(syncthingServiceMock.getDbFile, 'test-app', 'appdata/only-here'); } finally { publish.restore(); } }); + // Synced from the cluster since, this node's record is the cluster's version and + // no longer a change of its own. + it('reads the list again once the item that answered no longer does', async () => { + readyStandby(); + changes({ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }); + await stateMachine.manageFolderSyncState(mockParams); + changes({ name: 'appdata/other', localFlags: CHANGED, version: ['PRIMARY:5', 'LOCAL-D:9'] }); + syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ + local: { localFlags: 16, version: ['PRIMARY:6'] }, + }); + + await stateMachine.manageFolderSyncState(mockParams); + + sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + it('is neither reverted nor promoted while which changes others hold cannot be read', async () => { pollutedCandidate(); - syncthingServiceMock.getDbFile.rejects(new Error('connect ECONNREFUSED')); + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); const result = await stateMachine.manageFolderSyncState(mockParams); From 1670c52a789cf82006cafdc55e3b6a176b29575e Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:17:37 +0100 Subject: [PATCH 294/369] fix(syncthing): whether a folder's local changes hold the only copy of another device's data is checked off the monitor pass, and acted on only while the folder is unchanged The check reads the folder's list of local changes, asks about each change to data another device wrote - eight at a time, stopping at the first only copy - and leaves its answer on the folder's entry. One check runs at a time per node; the pass never waits for one. A folder whose changes have no answer yet is neither reverted nor promoted. An answer is stamped with this node's sequence, each other device's sequence and the devices the folder is shared with, and a pass acts on it only while the folder still carries that stamp. A device leaving the folder moves neither sequence, and is the moment a change it held becomes the only copy, so the shared devices are part of the stamp. An answer finishing after the folder's entry was replaced is dropped. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 154 ++++++++++++ .../syncthingFolderStateMachine.js | 116 +++------ test-infra/runner/framework/node-client.js | 1 + ...-went-quiet-and-then-died-is-taken-over.js | 5 - tests/unit/localChangeCheck.test.js | 144 +++++++++++ .../unit/syncthingFolderStateMachine.test.js | 227 +++++++++++++----- 6 files changed, 495 insertions(+), 152 deletions(-) create mode 100644 ZelBack/src/services/appMonitoring/localChangeCheck.js create mode 100644 tests/unit/localChangeCheck.test.js diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js new file mode 100644 index 0000000000..9bdcc8c0f0 --- /dev/null +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -0,0 +1,154 @@ +const log = require('../../lib/log'); +const syncthingService = require('../syncthingService'); +const fluxEventBus = require('../utils/fluxEventBus'); + +// Whether a receive-only folder's local changes include the only copy of data another +// device wrote, decided off the monitor pass. +// +// The answer takes a request per change to another device's data, and an app decides +// how many of those there are: a standby that rewrote its copy of a large folder has +// one for every file. Asked inside the pass, that is paid by every app on the node. +// Here a check runs on its own, one folder at a time per node, and the pass reads the +// answer it left. +// +// AN ANSWER HOLDS FOR THE FOLDER IT WAS ASKED ABOUT, AND NO OTHER. It is stamped with +// the folder as syncthing described it when the check was asked for: this node's +// sequence, each other device's sequence, and the devices the folder is shared with. +// A local write moves the first, another device's index the second, and a device +// leaving the folder - the moment a change held elsewhere becomes the only copy - +// changes the third. The pass acts on an answer only while the folder still carries +// its stamp. + +// syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const LOCAL_CHANGE_PAGE_SIZE = 1000; +// How many items are asked about at once. +const FILE_REQUESTS_AT_ONCE = 8; + +// Checks asked for and not started: { folderId, entry, stamp, isCurrent }. +const waiting = []; +let running = null; + +/** + * The folder as syncthing describes it, in the form an answer is stamped with. + * @param {object} status The folder's db/status: `sequence` and `remoteSequence`. + * @param {Array<{deviceID: string}>} devices The devices the folder is shared with. + * @returns {string} + */ +function stampOf(status, devices) { + const remote = Object.entries(status?.remoteSequence ?? {}).sort(([a], [b]) => a.localeCompare(b)); + const shared = devices.map((device) => device.deviceID).sort(); + return JSON.stringify({ sequence: status?.sequence ?? null, remote, shared }); +} + +/** + * The first of the named items whose cluster version is this node's own change: no + * other device holds a valid version of it. + * @param {string} folderId + * @param {string[]} names + * @returns {Promise} + */ +async function firstHeldOnlyHere(folderId, names) { + for (let start = 0; start < names.length; start += FILE_REQUESTS_AT_ONCE) { + const batch = names.slice(start, start + FILE_REQUESTS_AT_ONCE); + // eslint-disable-next-line no-await-in-loop + const answers = await Promise.all(batch.map((name) => syncthingService.getDbFile(folderId, name))); + // eslint-disable-next-line no-bitwise + const index = answers.findIndex((answer) => ((answer?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0); + if (index >= 0) return batch[index]; + } + return undefined; +} + +/** + * A local change that is the only copy of data from another device: its history names + * another device, so the data came from elsewhere, and no other device holds a valid + * version of it. syncthing reverts such an item by deleting it - the only copy of data + * the app wrote elsewhere - and a directory that is not empty cannot be deleted, so + * that revert fails on every pass. A change whose history is this node's alone was + * made here, never by the app's writer, and is not asked about. + * @param {string} folderId + * @returns {Promise} the item's name; false when there is none; null + * when the changes could not be read + */ +async function findOnlyCopy(folderId) { + try { + const deviceId = await syncthingService.getDeviceId(); + if (!deviceId) return null; + const ownCounter = `${deviceId.substring(0, 7)}:`; + for (let page = 1; ; page += 1) { + // eslint-disable-next-line no-await-in-loop + const batch = (await syncthingService.getDbLocalChanged(folderId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; + if (!Array.isArray(batch)) return page > 1 ? false : null; + const fromElsewhere = batch + .filter((item) => (item?.version ?? []).some((counter) => !counter.startsWith(ownCounter))) + .map((item) => item.name); + // eslint-disable-next-line no-await-in-loop + const found = await firstHeldOnlyHere(folderId, fromElsewhere); + if (found !== undefined) return found; + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return false; + } + } catch (error) { + log.warn(`localChangeCheck - ${folderId}: ${error.message}`); + return null; + } +} + +/** + * Runs one check and leaves its answer on the folder's entry, unless the entry has been + * replaced since it was asked for: the folder it describes is gone. + * @param {{folderId: string, entry: object, stamp: string, isCurrent: Function}} check + */ +async function runCheck(check) { + const onlyCopy = await findOnlyCopy(check.folderId); + if (onlyCopy === null || !check.isCurrent()) return; + // eslint-disable-next-line no-param-reassign + check.entry.localChangeCheck = { stamp: check.stamp, onlyCopy }; + fluxEventBus.publish('syncthing:localChangesChecked', { folder: check.folderId, onlyCopy: Boolean(onlyCopy) }); +} + +async function runWaitingChecks() { + while (waiting.length) { + running = waiting.shift(); + // eslint-disable-next-line no-await-in-loop + await runCheck(running); + } + running = null; +} + +/** + * Asks for a folder's local changes to be checked against the folder as `stamp` + * describes it. A check already asked for the folder is replaced; one running for the + * same folder, entry and stamp is left to finish. + * @param {string} folderId + * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache; the answer + * is left on it as `localChangeCheck`. + * @param {string} stamp stampOf the folder now. + * @param {Function} isCurrent Whether `entry` is still the folder's entry. + */ +function request(folderId, entry, stamp, isCurrent) { + if (running?.folderId === folderId && running.entry === entry && running.stamp === stamp) return; + const check = { folderId, entry, stamp, isCurrent }; + const index = waiting.findIndex((queued) => queued.folderId === folderId); + if (index >= 0) waiting[index] = check; + else waiting.push(check); + if (!running) runWaitingChecks(); +} + +/** + * The answer left for a folder's entry, if it was made against the folder as it is now. + * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache. + * @param {string} stamp stampOf the folder now. + * @returns {{onlyCopy: string|false}|null} + */ +function answerFor(entry, stamp) { + const answer = entry?.localChangeCheck; + return answer && answer.stamp === stamp ? answer : null; +} + +module.exports = { + answerFor, + findOnlyCopy, + request, + stampOf, +}; diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 8475be5d2d..e71800e788 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -29,6 +29,7 @@ const { const { isPathMounted } = require('../utils/volumeService'); const globalState = require('../utils/globalState'); const folderStates = require('./folderStates'); +const localChangeCheck = require('./localChangeCheck'); const { isSyncedRootName } = require('../appSystem/volumeReservedNames'); const primaryRole = require('../appLifecycle/primaryRole'); const { componentStateOnOtherHolders } = require('./peerComponent'); @@ -549,7 +550,7 @@ async function probeFolderSyncCompletion(folderId) { // so a folder of empty directories would read as holding data. const { globalFileBytes: globalBytes = 0, globalFiles = 0, inSyncFileBytes: inSyncBytes = 0, state, - receiveOnlyChangedFiles = 0, receiveOnlyChangedDirectories = 0, + receiveOnlyChangedFiles = 0, receiveOnlyChangedDirectories = 0, sequence, remoteSequence, } = await syncthingService.getDbStatusInFileBytes(folderId); const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; @@ -568,6 +569,9 @@ async function probeFolderSyncCompletion(folderId) { // Directories are counted apart from files. A directory whose owner or mode // differs from the cluster's is a local change like any other. receiveOnlyChangedDirectories, + // Where this node's index and each other device's stood when the status was read. + sequence, + remoteSequence, // An EMPTY global index (globalBytes 0) means "unknown / not yet synced", // never "done": a node holding the only copy before its peers reconnect // reads globalBytes 0, and syncPercentage defaults to 100 there (vacuous). @@ -1289,72 +1293,6 @@ async function nudgeFolderDevices(folderId) { } } -// syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. -const RECEIVE_ONLY_CHANGE_FLAG = 8; -// syncthing's FlagLocalGlobal: this device's record of the item is the cluster's version. -const GLOBAL_VERSION_FLAG = 16; -const LOCAL_CHANGE_PAGE_SIZE = 1000; - -/** - * Whether this node's record of an item is a local change to data from another device - * that no other device holds. A receive-only change is the cluster's version only when - * no other device holds a valid version of the item, and its history names another - * device when the data came from elsewhere. syncthing reverts such an item by deleting - * it - the only copy of data the app wrote elsewhere - and a directory that is not - * empty cannot be deleted, so that revert fails on every pass. - * - * An item whose history is this node's alone was made here, never by the app's - * writer: reverting deletes it, as it should. - * @param {object} item This node's record of the item, as db/localchanged and db/file's - * `local` give it. - * @param {string} ownCounter This node's version-vector prefix, `:`. - * @returns {boolean} - */ -function isOnlyCopyHere(item, ownCounter) { - const flags = item?.localFlags ?? 0; - // eslint-disable-next-line no-bitwise - const changedHere = Boolean(flags & RECEIVE_ONLY_CHANGE_FLAG); - // eslint-disable-next-line no-bitwise - const clusterVersion = Boolean(flags & GLOBAL_VERSION_FLAG); - const fromElsewhere = (item?.version ?? []).some((counter) => !counter.startsWith(ownCounter)); - return changedHere && clusterVersion && fromElsewhere; -} - -/** - * A local change in a receive-only folder that is the only copy of data from another - * device (isOnlyCopyHere), read from the folder's own list of local changes. - * - * `known` is the item that answered last time. It is asked about first, so a folder - * keeping its changes costs one request a pass while that item still answers; the - * list is read only when it does not. - * @param {string} appId - The Syncthing folder ID - * @param {string} [known] - The item that answered last time - * @returns {Promise} the item's name; false when there is none; null - * when the changes could not be read - */ -async function localChangeHeldOnlyHere(appId, known) { - try { - const deviceId = await syncthingService.getDeviceId(); - if (!deviceId) return null; - const ownCounter = `${deviceId.substring(0, 7)}:`; - if (known) { - const answer = await syncthingService.getDbFile(appId, known).catch(() => null); - if (isOnlyCopyHere(answer?.local, ownCounter)) return known; - } - for (let page = 1; ; page += 1) { - // eslint-disable-next-line no-await-in-loop - const batch = (await syncthingService.getDbLocalChanged(appId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; - if (!Array.isArray(batch)) return page > 1 ? false : null; - const found = batch.find((item) => isOnlyCopyHere(item, ownCounter)); - if (found) return found.name; - if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return false; - } - } catch (error) { - log.warn(`localChangeHeldOnlyHere - ${appId}: ${error.message}`); - return null; - } -} - /** * Reverts a synced receiveonly folder's local changes to the cluster's version. * @@ -1368,35 +1306,45 @@ async function localChangeHeldOnlyHere(appId, known) { * only copy of it left, and db/revert would delete it. A folder holding one is not * reverted - db/revert takes the whole folder, so its other changes stay too - and * its changes do not hold the node back from taking over: promoted, it publishes them. + * Which of the two a folder is comes from localChangeCheck, and is acted on only while + * the folder is still the one it was checked against. Until then the folder is neither + * reverted nor promoted. * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status - * @param {Object} [folderCache] - The folder's cache entry: `keepingChanges` names the - * item that is the only copy while the folder keeps its changes, so keeping them is said - * once, when it starts, and the item is asked about first on the next pass + * @param {Object} folderCache - The folder's entry in receiveOnlySyncthingAppsCache: it + * carries the check's answer, and `keepingChanges` marks a folder already said to be + * keeping its changes, so that is said once, when it starts + * @param {Function} isCurrent - Whether `folderCache` is still the folder's entry * @returns {Promise} True when the folder has local changes it does not - * keep: a revert was asked for, or which changes other devices hold could not be read + * keep: a revert was asked for, or which changes other devices hold is not yet known */ -async function revertLocalChangesIfSynced(appId, syncStatus, folderCache) { - const cache = folderCache || {}; +async function revertLocalChangesIfSynced(appId, syncStatus, folderCache, isCurrent) { + const cache = folderCache; if (!syncStatus?.isSynced) return false; const changed = (syncStatus.receiveOnlyChangedFiles || 0) + (syncStatus.receiveOnlyChangedDirectories || 0); if (changed === 0) { delete cache.keepingChanges; + delete cache.localChangeCheck; return false; } const files = syncStatus.receiveOnlyChangedFiles || 0; const directories = syncStatus.receiveOnlyChangedDirectories || 0; - const heldOnlyHere = await localChangeHeldOnlyHere(appId, cache.keepingChanges); - if (heldOnlyHere === null) { - log.warn(`revertLocalChangesIfSynced - ${appId}: could not read which of its ${changed} local change(s) other devices hold; neither reverting nor promoting this pass`); + const devices = (await syncthingService.getConfigFolders(appId).catch(() => null))?.devices; + if (!Array.isArray(devices)) { + log.warn(`revertLocalChangesIfSynced - ${appId}: could not read which devices share the folder; neither reverting nor promoting this pass`); + return true; + } + const stamp = localChangeCheck.stampOf(syncStatus, devices); + const answer = localChangeCheck.answerFor(cache, stamp); + if (!answer) { + localChangeCheck.request(appId, cache, stamp, isCurrent); return true; } - if (heldOnlyHere) { - const keeping = Boolean(cache.keepingChanges); - cache.keepingChanges = heldOnlyHere; - if (keeping) return false; - log.warn(`revertLocalChangesIfSynced - ${appId} holds a local change to data from another device that no other device holds (${files} files, ${directories} directories changed); keeping its changes, a revert would delete it`); + if (answer.onlyCopy) { + if (cache.keepingChanges) return false; + cache.keepingChanges = true; + log.warn(`revertLocalChangesIfSynced - ${appId} holds a local change to data from another device that no other device holds (${answer.onlyCopy}; ${files} files, ${directories} directories changed); keeping its changes, a revert would delete it`); fluxEventBus.publish('syncthing:localChangesKept', { folder: appId, files, directories }); return false; } @@ -1480,6 +1428,7 @@ async function handleReceiveOnlyTransition(params) { syncthingFolder, liveness, electedSiblings = [], + receiveOnlySyncthingAppsCache, } = params; const folderPath = syncthingFolder.path || `${appsFolder}${appId}/appdata`; @@ -1814,7 +1763,7 @@ async function handleReceiveOnlyTransition(params) { // a receiveonly folder) and promote on a later cycle once verifiably clean. Changes // no other device holds are kept, and promoted with the rest of this node's data. // The leader path above is exempt by design - the leader's local data IS the seed. - if (await revertLocalChangesIfSynced(appId, syncStatus, cache)) { + if (await revertLocalChangesIfSynced(appId, syncStatus, cache, () => receiveOnlySyncthingAppsCache.get(appId) === cache)) { return { syncthingFolder, cache }; } if (syncStatus.isSynced) { @@ -2084,6 +2033,7 @@ async function manageFolderSyncState(params) { syncthingFolder, liveness, electedSiblings, + receiveOnlySyncthingAppsCache, }); return result; } @@ -2130,7 +2080,7 @@ async function manageFolderSyncState(params) { // deviation (an owner or mode included) is never overwritten by syncthing // in a receiveonly folder, and the primary counts it as outstanding. if (syncFolder?.type === 'receiveonly') { - await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId), cache); + await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId), cache, () => receiveOnlySyncthingAppsCache.get(appId) === cache); } return { syncthingFolder, cache }; } diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 3fb847a9fa..8d9ccd436e 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -289,6 +289,7 @@ export function nodeClient(nodeNum) { 'syncthing:namesVisible', 'syncthing:localChangesReverted', 'syncthing:localChangesKept', + 'syncthing:localChangesChecked', 'syncthing:devicesResumed', 'pm2:killTimeoutRaiseFailed', 'pm2:killTimeoutUnchanged', diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index 055805ef60..ecd92d9d18 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -42,8 +42,6 @@ const MONITOR_PASSES_MS = 30000; const TAKEOVER_MS = 10 * 60 * 1000; // syncthing's FlagLocalReceiveOnly const RECEIVE_ONLY_CHANGE_FLAG = 8; -// syncthing's FlagLocalGlobal -const GLOBAL_VERSION_FLAG = 16; const SAVE = 'appdata/save-b'; const SAVE_CONTENT = 'written by the primary'; @@ -121,9 +119,6 @@ describe('a primary that went quiet and then died is taken over', function () { }, { timeout: 60000, interval: 2000, label: 'the file and the directory are changes no other device holds' }); expect(record.local.version.some((counter) => counter.startsWith(`${primaryDevice.substring(0, 7)}:`)), 'the changed file carries no history from the primary, so it is not the primary\'s data').to.equal(true); - // eslint-disable-next-line no-bitwise - expect((record.local.localFlags & GLOBAL_VERSION_FLAG) !== 0, - 'the standby\'s own record of the change is not marked the cluster\'s version, so its list of local changes cannot say no other device holds it').to.equal(true); expect(heldOnlyHere(await getFileInfo(client(A), folder, 'appdata/seed-data')), 'the unchanged file is held only here too, so the standby\'s copy no longer reads as synced').to.equal(false); diff --git a/tests/unit/localChangeCheck.test.js b/tests/unit/localChangeCheck.test.js new file mode 100644 index 0000000000..592ca5d133 --- /dev/null +++ b/tests/unit/localChangeCheck.test.js @@ -0,0 +1,144 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); +const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); + +const syncthingServiceMock = { + getDeviceId: sinon.stub(), + getDbLocalChanged: sinon.stub(), + getDbFile: sinon.stub(), +}; +const localChangeCheck = proxyquire('../../ZelBack/src/services/appMonitoring/localChangeCheck', { + '../syncthingService': syncthingServiceMock, +}); + +async function settled() { + for (let i = 0; i < 50; i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } +} + +// A read that waits until the test lets it answer. +function held(answer) { + let release; + const promise = new Promise((resolve) => { release = () => resolve(answer); }); + return { promise, release }; +} + +describe('localChangeCheck tests', () => { + beforeEach(() => { + syncthingServiceMock.getDeviceId.reset(); + syncthingServiceMock.getDeviceId.resolves('LOCAL-DEVICE'); + syncthingServiceMock.getDbLocalChanged.reset(); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [] }); + syncthingServiceMock.getDbFile.reset(); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 } }); + }); + + describe('stampOf', () => { + it('is the same for the same folder, whatever order syncthing lists it in', () => { + const one = localChangeCheck.stampOf({ sequence: 3, remoteSequence: { B: 2, A: 1 } }, [{ deviceID: 'Y' }, { deviceID: 'X' }]); + const two = localChangeCheck.stampOf({ sequence: 3, remoteSequence: { A: 1, B: 2 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]); + + expect(one).to.equal(two); + }); + + [ + ['this node\'s sequence', { sequence: 4, remoteSequence: { A: 1 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]], + ['another device\'s sequence', { sequence: 3, remoteSequence: { A: 2 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]], + ['the devices the folder is shared with', { sequence: 3, remoteSequence: { A: 1 } }, [{ deviceID: 'X' }]], + ].forEach(([what, status, devices]) => { + it(`changes with ${what}`, () => { + const before = localChangeCheck.stampOf({ sequence: 3, remoteSequence: { A: 1 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]); + + expect(localChangeCheck.stampOf(status, devices)).to.not.equal(before); + }); + }); + }); + + describe('request', () => { + it('runs one check at a time on the node', async () => { + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a').returns(first.promise); + const a = {}; + const b = {}; + + localChangeCheck.request('fluxa_a', a, 's1', () => true); + localChangeCheck.request('fluxb_b', b, 's1', () => true); + await settled(); + + sinon.assert.neverCalledWith(syncthingServiceMock.getDbLocalChanged, 'fluxb_b'); + first.release(); + await settled(); + expect(a.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false }); + expect(b.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false }); + }); + + it('replaces a check still waiting with the newer one for the same folder', async () => { + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a').returns(first.promise); + const b = {}; + + localChangeCheck.request('fluxa_a', {}, 's1', () => true); + localChangeCheck.request('fluxb_b', b, 'old', () => true); + localChangeCheck.request('fluxb_b', b, 'new', () => true); + first.release(); + await settled(); + + expect(syncthingServiceMock.getDbLocalChanged.withArgs('fluxb_b').callCount, 'the folder was checked once per stamp asked for').to.equal(1); + expect(b.localChangeCheck.stamp).to.equal('new'); + }); + + it('does not start again a check already running for the same folder and stamp', async () => { + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a').returns(first.promise); + const a = {}; + + localChangeCheck.request('fluxa_a', a, 's1', () => true); + await settled(); + localChangeCheck.request('fluxa_a', a, 's1', () => true); + first.release(); + await settled(); + + sinon.assert.calledOnce(syncthingServiceMock.getDbLocalChanged); + }); + + it('says when it has answered', async () => { + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + localChangeCheck.request('fluxa_a', {}, 's1', () => true); + await settled(); + + sinon.assert.calledWithExactly(publish, 'syncthing:localChangesChecked', { folder: 'fluxa_a', onlyCopy: false }); + } finally { + publish.restore(); + } + }); + }); + + describe('findOnlyCopy', () => { + it('asks about the items another device wrote, at most eight at once, and stops at the first only copy', async () => { + const files = Array.from({ length: 20 }, (_, k) => ({ name: `f${k}`, version: ['PRIMARY:1'] })); + syncthingServiceMock.getDbLocalChanged.resolves({ files }); + syncthingServiceMock.getDbFile.withArgs('fluxa_a', 'f9').resolves({ global: { localFlags: 8 } }); + + expect(await localChangeCheck.findOnlyCopy('fluxa_a')).to.equal('f9'); + expect(syncthingServiceMock.getDbFile.callCount, 'asked past the batch that held it').to.equal(16); + }); + + it('asks nothing about an item only this node has written', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); + + expect(await localChangeCheck.findOnlyCopy('fluxa_a')).to.equal(false); + sinon.assert.notCalled(syncthingServiceMock.getDbFile); + }); + + it('cannot say when an item cannot be read', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1'] }] }); + syncthingServiceMock.getDbFile.rejects(new Error('404 No such object in the index')); + + expect(await localChangeCheck.findOnlyCopy('fluxa_a')).to.equal(null); + }); + }); +}); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 96571e850c..32de669143 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -24,6 +24,7 @@ const syncthingServiceMock = { // reports 100% complete. getDeviceId: sinon.stub().resolves('LOCAL-DEVICE'), getConfigDevices: sinon.stub(), + getConfigFolders: sinon.stub(), dbRevert: sinon.stub(), scanFolder: sinon.stub(), getDeviceStats: sinon.stub(), @@ -175,7 +176,20 @@ const peerComponentMock = { componentStateOnOtherHolders: sinon.stub(), }; +const localChangeCheck = proxyquire('../../ZelBack/src/services/appMonitoring/localChangeCheck', { + '../syncthingService': syncthingServiceMock, +}); + +// Lets every check the passes asked for run to its end. +async function checksSettled() { + for (let i = 0; i < 50; i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } +} + const stateMachine = proxyquire('../../ZelBack/src/services/appMonitoring/syncthingFolderStateMachine', { + './localChangeCheck': localChangeCheck, '../dockerService': dockerServiceMock, '../syncthingService': syncthingServiceMock, '../serviceHelper': serviceHelperMock, @@ -220,6 +234,8 @@ describe('syncthingFolderStateMachine tests', () => { // default: one local change, whose cluster version another device holds syncthingServiceMock.getDbLocalChanged.reset(); syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'appdata/changed' }] }); + syncthingServiceMock.getConfigFolders.reset(); + syncthingServiceMock.getConfigFolders.resolves({ id: 'test-app', devices: [{ deviceID: 'LOCAL-DEVICE' }, { deviceID: 'PRIMARY-DEVICE' }] }); syncthingServiceMock.getDbFile.reset(); syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 }, local: { localFlags: 8 } }); syncthingServiceMock.scanFolder.reset(); @@ -941,11 +957,14 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder, 'a g: folder\'s type is its primary role\'s').to.not.have.property('type'); - expect(result.cache).to.deep.equal({ restarted: true }); + expect(result.cache).to.include({ restarted: true }); sinon.assert.calledWithExactly(publish, 'syncthing:localChangesReverted', { folder: 'test-app', files: 0, directories: 1 }); } finally { publish.restore(); @@ -968,6 +987,9 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); await stateMachine.manageFolderSyncState(mockParams); sinon.assert.calledOnce(syncthingServiceMock.dbRevert); @@ -2321,6 +2343,9 @@ describe('syncthingFolderStateMachine tests', () => { receiveOnlyChangedBytes: 555, }); + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); @@ -2332,97 +2357,192 @@ describe('syncthingFolderStateMachine tests', () => { // A change whose cluster version is this node's own is held by no other device: // syncthing reverts it by deleting it. One to data another device wrote is the // only copy of that data, so it is kept and promoted with the rest; one this node - // made itself is reverted, as any pollution is. Each is decided from the folder's - // list of local changes: syncthing marks the entry that is the cluster's version. + // made itself is reverted, as any pollution is. The check runs off the pass, and + // its answer is acted on only while the folder is the one it was checked against. describe('a local change no other device holds', () => { - const CHANGED = 8; - const CHANGED_AND_GLOBAL = 8 | 16; + const FROM_PRIMARY = ['PRIMARY:5', 'LOCAL-D:9']; const changes = (...files) => syncthingServiceMock.getDbLocalChanged.resolves({ files }); + const onlyHere = (name) => syncthingServiceMock.getDbFile.withArgs('test-app', name).resolves({ global: { localFlags: 8 }, local: { localFlags: 8 } }); + const heldElsewhere = (name) => syncthingServiceMock.getDbFile.withArgs('test-app', name).resolves({ global: { localFlags: 0 }, local: { localFlags: 8 } }); + const status = (over = {}) => syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, + sequence: 10, remoteSequence: { 'PRIMARY-DEVICE': 5 }, ...over, + }); + const sharedWith = (...devices) => syncthingServiceMock.getConfigFolders.resolves({ id: 'test-app', devices: devices.map((deviceID) => ({ deviceID })) }); const pollutedCandidate = () => { mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1 }); mockParams.appLocation.resolves([ { ip: '10.0.0.0:16127', runningSince: null, broadcastedAt: 1000 }, { ip: '10.0.0.1:16127', runningSince: null, broadcastedAt: 1000 }, ]); - syncthingServiceMock.getDbStatus.resolves({ - globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, - }); + status(); }; const readyStandby = () => { mockParams.containerDataFlags = 'g'; mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); - syncthingServiceMock.getDbStatus.resolves({ - globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, - }); + status(); + }; + const pass = () => stateMachine.manageFolderSyncState(mockParams); + const checkedPass = async () => { + await pass(); + await checksSettled(); + return pass(); }; - it('is kept, and the folder is promoted, decided from the list alone', async () => { + it('is neither reverted nor promoted before the check has answered', async () => { pollutedCandidate(); - changes({ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); - const result = await stateMachine.manageFolderSyncState(mockParams); + const result = await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + }); + + it('is kept, and the folder is promoted', async () => { + pollutedCandidate(); + changes({ name: 'appdata/only-here', version: FROM_PRIMARY }); + onlyHere('appdata/only-here'); + + const result = await checkedPass(); sinon.assert.notCalled(syncthingServiceMock.dbRevert); - sinon.assert.notCalled(syncthingServiceMock.getDbFile); expect(result.cache.restarted, 'a change only this node holds kept the folder from promoting').to.be.true; }); - it('is reverted when this node made it', async () => { + it('is reverted when this node made it, asking nothing about it', async () => { pollutedCandidate(); - changes({ name: 'appdata/made-here', localFlags: CHANGED_AND_GLOBAL, version: ['LOCAL-D:9'] }); + changes({ name: 'appdata/made-here', version: ['LOCAL-D:9'] }); - const result = await stateMachine.manageFolderSyncState(mockParams); + const result = await checkedPass(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + sinon.assert.notCalled(syncthingServiceMock.getDbFile); expect(result.cache.restarted).to.be.false; }); it('is reverted when another device holds the cluster\'s version', async () => { pollutedCandidate(); - changes({ name: 'appdata/held', localFlags: CHANGED, version: ['PRIMARY:5', 'LOCAL-D:9'] }); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); - await stateMachine.manageFolderSyncState(mockParams); + await checkedPass(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); }); it('is found behind changes other devices hold', async () => { pollutedCandidate(); - changes( - { name: 'appdata/held', localFlags: CHANGED, version: ['PRIMARY:5'] }, - { name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5'] }, - ); + changes({ name: 'appdata/held', version: FROM_PRIMARY }, { name: 'appdata/only-here', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + onlyHere('appdata/only-here'); - await stateMachine.manageFolderSyncState(mockParams); + await checkedPass(); sinon.assert.notCalled(syncthingServiceMock.dbRevert); }); it('is found on a later page', async () => { pollutedCandidate(); - const held = Array.from({ length: 1000 }, (_, k) => ({ name: `appdata/held-${k}`, localFlags: CHANGED, version: ['PRIMARY:5'] })); + const held = Array.from({ length: 1000 }, (_, k) => ({ name: `appdata/held-${k}`, version: FROM_PRIMARY })); syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 1).resolves({ files: held }); - syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 2).resolves({ - files: [{ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5'] }], + syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 2).resolves({ files: [{ name: 'appdata/only-here', version: FROM_PRIMARY }] }); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 } }); + onlyHere('appdata/only-here'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + // The primary dies and is dropped from the folder: a change it held is now the + // only copy. Neither counter moves when a device leaves, so the device list is + // what makes the answer from before stale. + it('is not reverted on an answer from before the primary left the folder', async () => { + pollutedCandidate(); + changes({ name: 'appdata/save', version: FROM_PRIMARY }); + heldElsewhere('appdata/save'); + await pass(); + await checksSettled(); + sharedWith('LOCAL-DEVICE'); + onlyHere('appdata/save'); + + await pass(); + await checksSettled(); + const result = await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted, 'the only copy, kept, is promoted').to.be.true; + }); + + [ + ['a local write', { sequence: 11 }], + ['another device\'s index', { remoteSequence: { 'PRIMARY-DEVICE': 6 } }], + ].forEach(([what, moved]) => { + it(`is checked again after ${what}, and not reverted on the answer from before it`, async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + status(moved); + + await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); }); + }); - await stateMachine.manageFolderSyncState(mockParams); + it('leaves no answer on a folder whose entry was replaced while it was checked', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + const checked = mockParams.receiveOnlySyncthingAppsCache.get('test-app'); + const replacement = { restarted: false, numberOfExecutions: 0 }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', replacement); + await checksSettled(); + + expect(checked, 'the replaced entry took the answer').to.not.have.property('localChangeCheck'); + expect(replacement).to.not.have.property('localChangeCheck'); + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('is neither reverted nor promoted while the changes cannot be read, and is asked about again', async () => { + pollutedCandidate(); + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); + + const result = await checkedPass(); sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); }); - it('is kept on a ready standby, said once, and asked about first on the next pass', async () => { + it('is neither reverted nor promoted while which devices share the folder cannot be read', async () => { + pollutedCandidate(); + syncthingServiceMock.getConfigFolders.rejects(new Error('connect ECONNREFUSED')); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.notCalled(syncthingServiceMock.getDbLocalChanged); + expect(result.cache.restarted).to.be.false; + }); + + it('is kept on a ready standby, said once, and costs nothing on the passes after', async () => { readyStandby(); - changes({ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }); - syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ - local: { localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }, - }); + changes({ name: 'appdata/only-here', version: FROM_PRIMARY }); + onlyHere('appdata/only-here'); const publish = sinon.stub(fluxEventBus, 'publish'); try { - await stateMachine.manageFolderSyncState(mockParams); - await stateMachine.manageFolderSyncState(mockParams); + await checkedPass(); + await pass(); + await pass(); sinon.assert.notCalled(syncthingServiceMock.dbRevert); sinon.assert.calledOnceWithExactly( @@ -2431,38 +2551,11 @@ describe('syncthingFolderStateMachine tests', () => { { folder: 'test-app', files: 0, directories: 1 }, ); sinon.assert.calledOnce(syncthingServiceMock.getDbLocalChanged); - sinon.assert.calledOnceWithExactly(syncthingServiceMock.getDbFile, 'test-app', 'appdata/only-here'); + sinon.assert.calledOnce(syncthingServiceMock.getDbFile); } finally { publish.restore(); } }); - - // Synced from the cluster since, this node's record is the cluster's version and - // no longer a change of its own. - it('reads the list again once the item that answered no longer does', async () => { - readyStandby(); - changes({ name: 'appdata/only-here', localFlags: CHANGED_AND_GLOBAL, version: ['PRIMARY:5', 'LOCAL-D:9'] }); - await stateMachine.manageFolderSyncState(mockParams); - changes({ name: 'appdata/other', localFlags: CHANGED, version: ['PRIMARY:5', 'LOCAL-D:9'] }); - syncthingServiceMock.getDbFile.withArgs('test-app', 'appdata/only-here').resolves({ - local: { localFlags: 16, version: ['PRIMARY:6'] }, - }); - - await stateMachine.manageFolderSyncState(mockParams); - - sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); - sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); - }); - - it('is neither reverted nor promoted while which changes others hold cannot be read', async () => { - pollutedCandidate(); - syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); - - const result = await stateMachine.manageFolderSyncState(mockParams); - - sinon.assert.notCalled(syncthingServiceMock.dbRevert); - expect(result.cache.restarted).to.be.false; - }); }); // Three properties of what counts as "holding", each able to hand the seed to the @@ -2711,6 +2804,9 @@ describe('syncthingFolderStateMachine tests', () => { globalBytes: 4096, globalFiles: 1, inSyncBytes: 4096, state: 'idle', receiveOnlyChangedFiles: 1, }); + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); sinon.assert.calledWith(syncthingServiceMock.dbRevert, 'test-app'); }); @@ -2956,6 +3052,9 @@ describe('syncthingFolderStateMachine tests', () => { receiveOnlyChangedDirectories: 2, }); + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); From 7f9d452db3d6557a3113a1bce8dce109cff91949 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:17:38 +0100 Subject: [PATCH 295/369] test(harness): the syncthing stub pages db/localchanged as syncthing does Co-Authored-By: Claude Opus 5.5 --- test-infra/syncthing-stub/index.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-infra/syncthing-stub/index.js b/test-infra/syncthing-stub/index.js index 05a9bebfb1..b5ef894b6d 100644 --- a/test-infra/syncthing-stub/index.js +++ b/test-infra/syncthing-stub/index.js @@ -759,8 +759,12 @@ app.get('/rest/db/localchanged', (req, res) => { if (ov?.statusUnreadable) return res.status(500).json({ error: 'simulated unreadable folder status' }); const folderCfg = reqState(req).folders.get(req.query.folder || ''); const isReceiveOnly = !folderCfg || folderCfg.type === 'receiveonly'; - const files = isReceiveOnly && Array.isArray(ov?.localChanged) ? ov.localChanged : []; - return res.json({ files, page: 1, perpage: 65536 }); + const all = isReceiveOnly && Array.isArray(ov?.localChanged) ? ov.localChanged : []; + // Paged as syncthing pages it: `page` is 1-based, `perpage` defaults to 65536. + const page = Math.max(1, Number(req.query.page) || 1); + const perpage = Math.max(1, Number(req.query.perpage) || 65536); + const files = all.slice((page - 1) * perpage, page * perpage); + return res.json({ files, page, perpage }); }); app.get('/rest/db/status', (req, res) => { From 36c3ae6d9163188c6b19e252e3df2162c37f53b8 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:20:00 +0100 Subject: [PATCH 296/369] test(harness): 3312 keeps a standby's change checked while the primary held it, once the primary is gone Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 1 + ZelBack/src/services/utils/fluxEventBus.js | 3 + ...eld-it-is-kept-once-the-primary-is-gone.js | 105 ++++++++++++++++++ 3 files changed, 109 insertions(+) create mode 100644 test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js index 9bdcc8c0f0..bb4680617b 100644 --- a/ZelBack/src/services/appMonitoring/localChangeCheck.js +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -101,6 +101,7 @@ async function findOnlyCopy(folderId) { */ async function runCheck(check) { const onlyCopy = await findOnlyCopy(check.folderId); + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER, check.folderId); if (onlyCopy === null || !check.isCurrent()) return; // eslint-disable-next-line no-param-reassign check.entry.localChangeCheck = { stamp: check.stamp, onlyCopy }; diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index 46eca167ad..f2dc345341 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -68,6 +68,9 @@ const Checkpoint = Object.freeze({ // The elected leader has read its sync status and has not yet checked the disk // it would seed from holds what that status claims. Keyed by folder id. SYNCTHING_BEFORE_SEED_CHECK: 'syncthing:beforeSeedCheck', + // A check of a receive-only folder's local changes has its answer and has not + // left it for the monitor pass. Keyed by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER: 'syncthing:beforeLocalChangeAnswer', // The syncthing monitor has reached a receive-only folder and not yet measured // what this node holds in it. Keyed by folder id. SYNCTHING_BEFORE_HOLDINGS: 'syncthing:beforeHoldings', diff --git a/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js b/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js new file mode 100644 index 0000000000..e40fbd8b2a --- /dev/null +++ b/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js @@ -0,0 +1,105 @@ +// weight: heavy +/* + * A standby's copy of a file the primary wrote changes while the primary still holds + * it. Whether the standby's changes hold the only copy of anything is checked off the + * monitor pass, and this check answers that they do not: the primary holds the file. + * + * The check is held before it leaves that answer, and the primary's machine dies and + * is dropped from the folder. From that moment the change is the only copy of the + * primary's data. The answer, released, describes the folder as it was; a revert on + * it deletes that copy. The standby keeps the change, and the primary's data is still + * on its disk. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { crashFluxos, execInContainer, stopAppContainer } from '../framework/container.js'; +import { + getDeviceId, getFolderConfig, getFileInfo, readPath, scanFolder, stopDaemon, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const BEFORE_ANSWER = 'syncthing:beforeLocalChangeAnswer'; +// syncthing's FlagLocalReceiveOnly +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const SAVE = 'appdata/save-b'; +const SAVE_CONTENT = 'written by the primary'; + +function step(what) { + console.log(`# 3312 ${new Date().toISOString()} ${what}`); +} + +describe('a change checked while the primary held it is kept once the primary is gone', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2echeckedstale${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let primaryDevice; + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + primaryDevice = await getDeviceId(client(B)); + const wrote = await execInContainer(client(B).container, `printf '${SAVE_CONTENT}' > ${root}/${SAVE}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on the primary: ${wrote.output}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 240000, interval: 3000, label: 'the primary\'s file reached the standby', + }); + }); + + after(async function () { + this.timeout(60000); + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('keeps the change, though the check answered while the primary held it', async function () { + this.timeout(1500000); + // eslint-disable-next-line no-bitwise + const heldOnlyHere = (record) => ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; + + step('the standby\'s copy of the primary\'s file changes, while the primary holds it'); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const changedAt = client(A).getLastEventId(); + const changed = await execInContainer(client(A).container, `chmod 600 ${root}/${SAVE}`); + expect(changed.exitCode, `could not change the standby's copy: ${changed.output}`).to.equal(0); + await scanFolder(client(A), folder); + expect(heldOnlyHere(await getFileInfo(client(A), folder, SAVE)), 'fixture: the primary does not hold the file').to.equal(false); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_ANSWER && d.key === folder, 300000, { afterId: changedAt }); + + step('the primary\'s machine dies and the standby drops it from the folder'); + await crashFluxos(client(B).container, { hold: true }); + await Promise.all([A, 2].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); + await stopAppContainer(client(B).container, appName, appName); + await stopDaemon(client(B)); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) + .some((d) => d.deviceID === primaryDevice), { + timeout: 600000, interval: 5000, label: 'the standby drops the dead primary from the folder', + }); + await waitFor(async () => heldOnlyHere(await getFileInfo(client(A), folder, SAVE)), { + timeout: 60000, interval: 2000, label: 'fixture: the change is now the only copy', + }); + + step('the check, answered before the primary left, is released'); + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + await client(A).waitForEvent('syncthing:localChangesKept', (d) => d.folder === folder, 300000, { afterId: releasedAt }); + + const reverted = client(A).getEventBuffer().some((e) => e.id > changedAt && e.event === 'syncthing:localChangesReverted' && e.data?.folder === folder); + expect(reverted, 'the standby reverted the only copy on an answer from before the primary left').to.equal(false); + expect(await readPath(client(A), `${root}/${SAVE}`), 'the primary\'s file is gone from the standby').to.equal(SAVE_CONTENT); + }); +}); From f09f9bc5f7c28b133f427f81a774c5c8b9a60015 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:21:35 +0100 Subject: [PATCH 297/369] test(harness): 3313 checks a standby's local changes off the pass, reverting only on an answer for the folder as it is Co-Authored-By: Claude Opus 5.5 --- ...-local-changes-are-checked-off-the-pass.js | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js diff --git a/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js new file mode 100644 index 0000000000..52cad81db6 --- /dev/null +++ b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js @@ -0,0 +1,125 @@ +// weight: heavy +/* + * Whether a standby's local changes hold the only copy of the primary's data is + * checked off the monitor pass, and acted on only while the folder is the one it was + * checked against. The primary is alive throughout, so every change here is held + * elsewhere and is reverted in the end; what is tested is when. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { crashFluxos, releaseFluxos, execInContainer } from '../framework/container.js'; +import { readPath, scanFolder } from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const BEFORE_ANSWER = 'syncthing:beforeLocalChangeAnswer'; +const FILES = 2000; +const PASSES_WHILE_HELD = 3; + +function step(what) { + console.log(`# 3313 ${new Date().toISOString()} ${what}`); +} + +describe('a standby\'s local changes are checked off the monitor pass', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2echeckoffpass${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const passes = () => client(A).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + const heldAt = (from) => client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_ANSWER && d.key === folder, 300000, { afterId: from }); + const event = (name, from) => client(A).waitForEvent(name, (d) => d.folder === folder, 300000, { afterId: from }); + const eventsSince = (name, from) => client(A).getEventBuffer().filter((e) => e.id > from && e.event === name && e.data?.folder === folder); + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + await sh(B, `mkdir -p ${root}/appdata/many && for i in $(seq 1 ${FILES}); do printf 'primary %s' $i > ${root}/appdata/many/f$i; done`); + await scanFolder(client(B), folder); + await waitFor(async () => (await sh(A, `ls ${root}/appdata/many | wc -l`)).trim() === String(FILES), { + timeout: 600000, interval: 5000, label: 'the primary\'s files reached the standby', + }); + }); + + after(async function () { + this.timeout(60000); + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('runs the standby\'s passes while it checks a rewrite of every file the primary wrote, then reverts it', async function () { + this.timeout(900000); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const from = client(A).getLastEventId(); + step(`the standby rewrites its copy of ${FILES} of the primary's files`); + await sh(A, `for i in $(seq 1 ${FILES}); do printf ' standby' >> ${root}/appdata/many/f$i; done`); + await scanFolder(client(A), folder); + await heldAt(from); + + const passesAtHold = await passes(); + await waitFor(async () => await passes() >= passesAtHold + PASSES_WHILE_HELD, { + timeout: 120000, interval: 2000, label: `${PASSES_WHILE_HELD} passes over the folder while its check is held`, + }); + expect(eventsSince('syncthing:localChangesReverted', from), 'reverted before the check answered').to.deep.equal([]); + + await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + await event('syncthing:localChangesReverted', from); + await waitFor(async () => (await readPath(client(A), `${root}/appdata/many/f1`)) === 'primary 1', { + timeout: 120000, interval: 2000, label: 'the standby holds the primary\'s copy again', + }); + }); + + it('does not revert on an answer from before a later write, and reverts once that write is checked', async function () { + this.timeout(900000); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const from = client(A).getLastEventId(); + await sh(A, `printf ' first' >> ${root}/appdata/many/f1`); + await scanFolder(client(A), folder); + await heldAt(from); + + step('a second write lands while the first is being checked'); + await sh(A, `printf ' second' >> ${root}/appdata/many/f2`); + await scanFolder(client(A), folder); + const passesAfterWrite = await passes(); + await waitFor(async () => await passes() >= passesAfterWrite + 2, { + timeout: 120000, interval: 2000, label: 'two passes have seen the second write', + }); + + await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + const reverted = await event('syncthing:localChangesReverted', from); + const checks = eventsSince('syncthing:localChangesChecked', from); + expect(checks.length, 'checks answered before the revert').to.be.at.least(2); + expect(reverted.id, 'reverted on the answer from before the second write').to.be.greaterThan(checks[1].id); + }); + + it('checks again after FluxOS restarts mid-check, and reverts', async function () { + this.timeout(900000); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const from = client(A).getLastEventId(); + await sh(A, `printf ' again' >> ${root}/appdata/many/f3`); + await scanFolder(client(A), folder); + await heldAt(from); + + step('the standby\'s FluxOS restarts with the check held'); + await crashFluxos(client(A).container, { hold: true }); + await releaseFluxos(client(A).container); + await waitFor(async () => (await readPath(client(A), `${root}/appdata/many/f3`)) === 'primary 3', { + timeout: 600000, interval: 5000, label: 'the restarted standby reverts to the primary\'s copy', + }); + }); +}); From 628e5b1459ba517685c2da6ca45b3e1164183e41 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:34:34 +0100 Subject: [PATCH 298/369] test(harness): 3312 and 3313 hold the next check before it starts, so the passes act on the answer from before the folder changed Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 1 + ZelBack/src/services/utils/fluxEventBus.js | 3 +++ ...eld-it-is-kept-once-the-primary-is-gone.js | 21 ++++++++++++++++--- ...-local-changes-are-checked-off-the-pass.js | 18 ++++++++++++---- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js index bb4680617b..720b8b5db0 100644 --- a/ZelBack/src/services/appMonitoring/localChangeCheck.js +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -100,6 +100,7 @@ async function findOnlyCopy(folderId) { * @param {{folderId: string, entry: object, stamp: string, isCurrent: Function}} check */ async function runCheck(check) { + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_CHECK, check.folderId); const onlyCopy = await findOnlyCopy(check.folderId); await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER, check.folderId); if (onlyCopy === null || !check.isCurrent()) return; diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index f2dc345341..1df515d472 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -68,6 +68,9 @@ const Checkpoint = Object.freeze({ // The elected leader has read its sync status and has not yet checked the disk // it would seed from holds what that status claims. Keyed by folder id. SYNCTHING_BEFORE_SEED_CHECK: 'syncthing:beforeSeedCheck', + // A check of a receive-only folder's local changes has been started and has not + // read anything. Keyed by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_CHECK: 'syncthing:beforeLocalChangeCheck', // A check of a receive-only folder's local changes has its answer and has not // left it for the monitor pass. Keyed by folder id. SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER: 'syncthing:beforeLocalChangeAnswer', diff --git a/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js b/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js index e40fbd8b2a..ae11c00130 100644 --- a/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js +++ b/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js @@ -7,8 +7,9 @@ * The check is held before it leaves that answer, and the primary's machine dies and * is dropped from the folder. From that moment the change is the only copy of the * primary's data. The answer, released, describes the folder as it was; a revert on - * it deletes that copy. The standby keeps the change, and the primary's data is still - * on its disk. + * it deletes that copy. The check the standby asks for next is held before it starts, + * so the old answer is the one its passes find. They revert nothing, and once that + * check runs the standby keeps the change, with the primary's data still on its disk. * * A = the standby, B = the primary, C = holds nothing. */ @@ -29,6 +30,8 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const SIGTERM_EXPIRY_S = 120; const BEFORE_ANSWER = 'syncthing:beforeLocalChangeAnswer'; +const BEFORE_CHECK = 'syncthing:beforeLocalChangeCheck'; +const PASSES_ON_THE_OLD_ANSWER = 3; // syncthing's FlagLocalReceiveOnly const RECEIVE_ONLY_CHANGE_FLAG = 8; const SAVE = 'appdata/save-b'; @@ -45,6 +48,7 @@ describe('a change checked while the primary held it is kept once the primary is const folder = `flux${appName}_${appName}`; const root = `/mnt/appdata/flux-apps/${folder}`; const client = (i) => fleet.env.clients[i]; + const passes = () => client(A).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); let primaryDevice; before(async function () { @@ -93,9 +97,20 @@ describe('a change checked while the primary held it is kept once the primary is timeout: 60000, interval: 2000, label: 'fixture: the change is now the only copy', }); - step('the check, answered before the primary left, is released'); + step('the check, answered before the primary left, is released; the next is held before it starts'); + await client(A).holdCheckpoint(BEFORE_CHECK, folder); const releasedAt = client(A).getLastEventId(); await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + await client(A).waitForEvent('syncthing:localChangesChecked', (d) => d.folder === folder && d.onlyCopy === false, 300000, { afterId: releasedAt }); + const passesOnOldAnswer = await passes(); + await waitFor(async () => await passes() >= passesOnOldAnswer + PASSES_ON_THE_OLD_ANSWER, { + timeout: 120000, interval: 2000, label: `${PASSES_ON_THE_OLD_ANSWER} passes with the answer from before the primary left`, + }); + const revertedOnOldAnswer = client(A).getEventBuffer().some((e) => e.id > releasedAt && e.event === 'syncthing:localChangesReverted' && e.data?.folder === folder); + expect(revertedOnOldAnswer, 'the standby reverted the only copy on the answer from before the primary left').to.equal(false); + + step('the check asked for since runs'); + await client(A).releaseCheckpoint(BEFORE_CHECK, folder); await client(A).waitForEvent('syncthing:localChangesKept', (d) => d.folder === folder, 300000, { afterId: releasedAt }); const reverted = client(A).getEventBuffer().some((e) => e.id > changedAt && e.event === 'syncthing:localChangesReverted' && e.data?.folder === folder); diff --git a/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js index 52cad81db6..5d05efe2f7 100644 --- a/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js +++ b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js @@ -21,6 +21,7 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const SIGTERM_EXPIRY_S = 120; const BEFORE_ANSWER = 'syncthing:beforeLocalChangeAnswer'; +const BEFORE_CHECK = 'syncthing:beforeLocalChangeCheck'; const FILES = 2000; const PASSES_WHILE_HELD = 3; @@ -100,11 +101,20 @@ describe('a standby\'s local changes are checked off the monitor pass', function timeout: 120000, interval: 2000, label: 'two passes have seen the second write', }); + // The check asked for since the second write is held before it starts, so the + // passes find the answer from before it. + await client(A).holdCheckpoint(BEFORE_CHECK, folder); + const releasedAt = client(A).getLastEventId(); await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); - const reverted = await event('syncthing:localChangesReverted', from); - const checks = eventsSince('syncthing:localChangesChecked', from); - expect(checks.length, 'checks answered before the revert').to.be.at.least(2); - expect(reverted.id, 'reverted on the answer from before the second write').to.be.greaterThan(checks[1].id); + await event('syncthing:localChangesChecked', releasedAt); + const passesOnOldAnswer = await passes(); + await waitFor(async () => await passes() >= passesOnOldAnswer + 3, { + timeout: 120000, interval: 2000, label: 'three passes with the answer from before the second write', + }); + expect(eventsSince('syncthing:localChangesReverted', from), 'reverted on the answer from before the second write').to.deep.equal([]); + + await client(A).releaseCheckpoint(BEFORE_CHECK, folder); + await event('syncthing:localChangesReverted', from); }); it('checks again after FluxOS restarts mid-check, and reverts', async function () { From e7831474f769b5c711f9ca1a97f541315d311108 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:50:29 +0100 Subject: [PATCH 299/369] test(harness): 3313 releases its checkpoints after each test Co-Authored-By: Claude Opus 5.5 --- ...13-a-standbys-local-changes-are-checked-off-the-pass.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js index 5d05efe2f7..f0f66d653e 100644 --- a/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js +++ b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js @@ -8,7 +8,7 @@ * A = the standby, B = the primary, C = holds nothing. */ import { - describe, it, before, after, + describe, it, before, after, afterEach, } from 'mocha'; import { expect } from 'chai'; import { @@ -56,6 +56,11 @@ describe('a standby\'s local changes are checked off the monitor pass', function }); }); + // A test that fails holding a check would hold the next test's checks too. + afterEach(async function () { + await client(A)?.releaseAllCheckpoints().catch(() => {}); + }); + after(async function () { this.timeout(60000); await client(A)?.releaseAllCheckpoints().catch(() => {}); From 13790ea0e84a5fa4d86b1482eec6a4c3a1c40b8c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:57:57 +0100 Subject: [PATCH 300/369] fix(syncthing): a standby's own changes are reverted only while another holder runs the app A change whose history is this node's alone was taken for junk and reverted, which deletes it when no other device holds it. A node that was the app's primary earlier holds such history for the data it wrote then; left the only holder, its change to a file of its own was deleted, and one to a directory of its own could never be reverted, so its folder never became ready to take over. A single-writer folder whose changes include one this node made is now reverted only while another holder runs the app: that holder is authoritative, and a change it does not have is this node's alone. While no holder runs it the changes are kept, and while that cannot be said the folder is neither reverted nor promoted. It is asked again on every pass that would revert, so a folder kept while no holder ran the app is reverted once one does. A change that is the only copy of another device's data is kept as before, and an r: folder, which every holder runs, is reverted as before. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 49 ++++---- .../syncthingFolderStateMachine.js | 83 +++++++++---- tests/unit/localChangeCheck.test.js | 30 +++-- .../unit/syncthingFolderStateMachine.test.js | 113 +++++++++++++++++- 4 files changed, 222 insertions(+), 53 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js index 720b8b5db0..7b0fe41a5a 100644 --- a/ZelBack/src/services/appMonitoring/localChangeCheck.js +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -61,32 +61,39 @@ async function firstHeldOnlyHere(folderId, names) { } /** - * A local change that is the only copy of data from another device: its history names - * another device, so the data came from elsewhere, and no other device holds a valid - * version of it. syncthing reverts such an item by deleting it - the only copy of data - * the app wrote elsewhere - and a directory that is not empty cannot be deleted, so - * that revert fails on every pass. A change whose history is this node's alone was - * made here, never by the app's writer, and is not asked about. + * What a receive-only folder's local changes are, for whether they may be reverted: + * - `onlyCopy`: a change that is the only copy of data from another device. Its + * history names another device, so the data came from elsewhere, and no other + * device holds a valid version of it. syncthing reverts such an item by deleting + * it, and a directory that is not empty cannot be deleted, so that revert fails on + * every pass. The first one found ends the check. + * - `madeHere`: whether any change's history is this node's alone. Such a change was + * made on this node, as a standby or earlier as the primary, and no other device's + * history says otherwise. It is not asked about: whether it may be reverted turns + * on whether another holder runs the app, not on what other devices hold. * @param {string} folderId - * @returns {Promise} the item's name; false when there is none; null - * when the changes could not be read + * @returns {Promise<{onlyCopy: string|false, madeHere: boolean}|null>} null when the + * changes could not be read */ -async function findOnlyCopy(folderId) { +async function examineLocalChanges(folderId) { try { const deviceId = await syncthingService.getDeviceId(); if (!deviceId) return null; const ownCounter = `${deviceId.substring(0, 7)}:`; + let madeHere = false; for (let page = 1; ; page += 1) { // eslint-disable-next-line no-await-in-loop const batch = (await syncthingService.getDbLocalChanged(folderId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; - if (!Array.isArray(batch)) return page > 1 ? false : null; - const fromElsewhere = batch - .filter((item) => (item?.version ?? []).some((counter) => !counter.startsWith(ownCounter))) - .map((item) => item.name); + if (!Array.isArray(batch)) return page > 1 ? { onlyCopy: false, madeHere } : null; + const fromElsewhere = []; + batch.forEach((item) => { + if ((item?.version ?? []).some((counter) => !counter.startsWith(ownCounter))) fromElsewhere.push(item.name); + else madeHere = true; + }); // eslint-disable-next-line no-await-in-loop const found = await firstHeldOnlyHere(folderId, fromElsewhere); - if (found !== undefined) return found; - if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return false; + if (found !== undefined) return { onlyCopy: found, madeHere }; + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return { onlyCopy: false, madeHere }; } } catch (error) { log.warn(`localChangeCheck - ${folderId}: ${error.message}`); @@ -101,12 +108,12 @@ async function findOnlyCopy(folderId) { */ async function runCheck(check) { await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_CHECK, check.folderId); - const onlyCopy = await findOnlyCopy(check.folderId); + const found = await examineLocalChanges(check.folderId); await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER, check.folderId); - if (onlyCopy === null || !check.isCurrent()) return; + if (found === null || !check.isCurrent()) return; // eslint-disable-next-line no-param-reassign - check.entry.localChangeCheck = { stamp: check.stamp, onlyCopy }; - fluxEventBus.publish('syncthing:localChangesChecked', { folder: check.folderId, onlyCopy: Boolean(onlyCopy) }); + check.entry.localChangeCheck = { stamp: check.stamp, ...found }; + fluxEventBus.publish('syncthing:localChangesChecked', { folder: check.folderId, onlyCopy: Boolean(found.onlyCopy), madeHere: found.madeHere }); } async function runWaitingChecks() { @@ -141,7 +148,7 @@ function request(folderId, entry, stamp, isCurrent) { * The answer left for a folder's entry, if it was made against the folder as it is now. * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache. * @param {string} stamp stampOf the folder now. - * @returns {{onlyCopy: string|false}|null} + * @returns {{onlyCopy: string|false, madeHere: boolean}|null} */ function answerFor(entry, stamp) { const answer = entry?.localChangeCheck; @@ -150,7 +157,7 @@ function answerFor(entry, stamp) { module.exports = { answerFor, - findOnlyCopy, + examineLocalChanges, request, stampOf, }; diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index e71800e788..33bf5f8e26 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -32,7 +32,7 @@ const folderStates = require('./folderStates'); const localChangeCheck = require('./localChangeCheck'); const { isSyncedRootName } = require('../appSystem/volumeReservedNames'); const primaryRole = require('../appLifecycle/primaryRole'); -const { componentStateOnOtherHolders } = require('./peerComponent'); +const { componentStateOnOtherHolders, PeerComponent } = require('./peerComponent'); const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); @@ -1302,12 +1302,19 @@ async function nudgeFolderDevices(folderId) { * version and deletes local-only items that are not ignored, so it runs only * against a complete global index (isSynced). * - * A change no other device holds, to data that came from another device, is the - * only copy of it left, and db/revert would delete it. A folder holding one is not - * reverted - db/revert takes the whole folder, so its other changes stay too - and - * its changes do not hold the node back from taking over: promoted, it publishes them. - * Which of the two a folder is comes from localChangeCheck, and is acted on only while - * the folder is still the one it was checked against. Until then the folder is neither + * db/revert takes the whole folder, and deletes every change no other device holds. + * A folder is reverted only when both say its changes are not the app's data: + * - none of them is the only copy of data another device wrote, and + * - another holder runs the app, when any of them is a change this node made itself. + * A holder running the app is authoritative, and a change it does not have is + * this node's alone. No holder running it, this node holds the app's data, its + * earlier writes as primary included, and a revert has nothing to restore them + * from. Whether a holder runs it is asked on every pass that would revert, so a + * folder kept while none did is reverted once one does. + * A kept folder's changes do not hold the node back from taking over: promoted, it + * publishes them. Which changes the folder holds comes from localChangeCheck, and is + * acted on only while the folder is still the one it was checked against. Until then, + * and while whether another holder runs the app cannot be said, the folder is neither * reverted nor promoted. * * @param {string} appId - The Syncthing folder ID @@ -1315,11 +1322,14 @@ async function nudgeFolderDevices(folderId) { * @param {Object} folderCache - The folder's entry in receiveOnlySyncthingAppsCache: it * carries the check's answer, and `keepingChanges` marks a folder already said to be * keeping its changes, so that is said once, when it starts - * @param {Function} isCurrent - Whether `folderCache` is still the folder's entry + * @param {Object} options + * @param {Function} options.isCurrent - Whether `folderCache` is still the folder's entry + * @param {Function|null} options.othersRunTheApp - What the app's other holders are doing + * with it, as a PeerComponent state; null when every holder runs it * @returns {Promise} True when the folder has local changes it does not - * keep: a revert was asked for, or which changes other devices hold is not yet known + * keep: a revert was asked for, or whether to revert cannot be said yet */ -async function revertLocalChangesIfSynced(appId, syncStatus, folderCache, isCurrent) { +async function revertLocalChangesIfSynced(appId, syncStatus, folderCache, { isCurrent, othersRunTheApp }) { const cache = folderCache; if (!syncStatus?.isSynced) return false; const changed = (syncStatus.receiveOnlyChangedFiles || 0) + (syncStatus.receiveOnlyChangedDirectories || 0); @@ -1341,13 +1351,32 @@ async function revertLocalChangesIfSynced(appId, syncStatus, folderCache, isCurr localChangeCheck.request(appId, cache, stamp, isCurrent); return true; } - if (answer.onlyCopy) { - if (cache.keepingChanges) return false; - cache.keepingChanges = true; - log.warn(`revertLocalChangesIfSynced - ${appId} holds a local change to data from another device that no other device holds (${answer.onlyCopy}; ${files} files, ${directories} directories changed); keeping its changes, a revert would delete it`); - fluxEventBus.publish('syncthing:localChangesKept', { folder: appId, files, directories }); + const keep = (reason, why) => { + if (cache.keepingChanges === reason) return false; + cache.keepingChanges = reason; + log.warn(`revertLocalChangesIfSynced - ${appId} ${why} (${files} files, ${directories} directories changed); keeping its changes, a revert would delete them`); + fluxEventBus.publish('syncthing:localChangesKept', { + folder: appId, files, directories, reason, + }); return false; + }; + if (answer.onlyCopy) { + delete cache.waitingOnHolders; + return keep('onlyCopy', `holds a local change to data from another device that no other device holds (${answer.onlyCopy})`); + } + if (answer.madeHere && othersRunTheApp) { + const elsewhere = await othersRunTheApp(); + if (elsewhere === PeerComponent.NOT_RUNNING) { + delete cache.waitingOnHolders; + return keep('noHolderRunsTheApp', 'holds changes it made itself while no other holder runs the app'); + } + if (elsewhere !== PeerComponent.RUNNING) { + if (!cache.waitingOnHolders) log.info(`revertLocalChangesIfSynced - ${appId}: whether another holder runs the app cannot be said; neither reverting nor promoting until it can`); + cache.waitingOnHolders = true; + return true; + } } + delete cache.waitingOnHolders; delete cache.keepingChanges; log.warn(`revertLocalChangesIfSynced - ${appId} is synced but the receive-only folder has ${changed} locally changed item(s) (${files} files, ${directories} directories); reverting to the cluster's version`); try { @@ -1429,6 +1458,7 @@ async function handleReceiveOnlyTransition(params) { liveness, electedSiblings = [], receiveOnlySyncthingAppsCache, + othersRunTheApp = null, } = params; const folderPath = syncthingFolder.path || `${appsFolder}${appId}/appdata`; @@ -1763,7 +1793,9 @@ async function handleReceiveOnlyTransition(params) { // a receiveonly folder) and promote on a later cycle once verifiably clean. Changes // no other device holds are kept, and promoted with the rest of this node's data. // The leader path above is exempt by design - the leader's local data IS the seed. - if (await revertLocalChangesIfSynced(appId, syncStatus, cache, () => receiveOnlySyncthingAppsCache.get(appId) === cache)) { + if (await revertLocalChangesIfSynced(appId, syncStatus, cache, { + isCurrent: () => receiveOnlySyncthingAppsCache.get(appId) === cache, othersRunTheApp, + })) { return { syncthingFolder, cache }; } if (syncStatus.isSynced) { @@ -1955,6 +1987,15 @@ async function manageFolderSyncState(params) { // passes from an event to know every holder has decided since it. fluxEventBus.count('syncthing:folderPass', appId, 'evaluated'); + // What the app's other holders are doing with its g: components. Every one of + // them is asked about: a holder running any of them is its primary. + const othersHold = () => componentStateOnOtherHolders(readAppLocation, localSocketAddr, { + appIds: [appId, ...electedSiblings], identifier, appName: installedAppName, liveness, logPrefix: 'manageFolderSyncState', + }); + // Whether a change only this node made, and no other device holds, is reverted + // turns on whether another holder runs the app. Every holder of an r: folder runs it. + const othersRunTheApp = containerDataFlags.includes('g') ? othersHold : null; + // Check if folder already exists and is in sendreceive mode const folderAlreadySyncing = syncFolder && syncFolder.type === 'sendreceive'; const sendingSingleWriter = folderAlreadySyncing && containerDataFlags.includes('g'); @@ -1971,11 +2012,6 @@ async function manageFolderSyncState(params) { // its process, and what it holds must not go out until the election says // it is primary again. log.info(`manageFolderSyncState - ${appId} is sendreceive and not held here, demoting until the election decides`); - // Every g: component of the app is asked about: a holder running any of - // them is its primary, and this folder's changes are not to go out beside it. - const othersHold = () => componentStateOnOtherHolders(readAppLocation, localSocketAddr, { - appIds: [appId, ...electedSiblings], identifier, appName: installedAppName, liveness, logPrefix: 'manageFolderSyncState', - }); await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_STANDBY_HOLD, appId); await primaryRole.holdAsStandby(identifier, appId, { othersHold }); return { syncthingFolder, cache: { restarted: false, numberOfExecutions: 0 } }; @@ -2034,6 +2070,7 @@ async function manageFolderSyncState(params) { liveness, electedSiblings, receiveOnlySyncthingAppsCache, + othersRunTheApp, }); return result; } @@ -2080,7 +2117,9 @@ async function manageFolderSyncState(params) { // deviation (an owner or mode included) is never overwritten by syncthing // in a receiveonly folder, and the primary counts it as outstanding. if (syncFolder?.type === 'receiveonly') { - await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId), cache, () => receiveOnlySyncthingAppsCache.get(appId) === cache); + await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId), cache, { + isCurrent: () => receiveOnlySyncthingAppsCache.get(appId) === cache, othersRunTheApp, + }); } return { syncthingFolder, cache }; } diff --git a/tests/unit/localChangeCheck.test.js b/tests/unit/localChangeCheck.test.js index 592ca5d133..616f8f9660 100644 --- a/tests/unit/localChangeCheck.test.js +++ b/tests/unit/localChangeCheck.test.js @@ -71,8 +71,8 @@ describe('localChangeCheck tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.getDbLocalChanged, 'fluxb_b'); first.release(); await settled(); - expect(a.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false }); - expect(b.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false }); + expect(a.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false, madeHere: false }); + expect(b.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false, madeHere: false }); }); it('replaces a check still waiting with the newer one for the same folder', async () => { @@ -110,35 +110,49 @@ describe('localChangeCheck tests', () => { localChangeCheck.request('fluxa_a', {}, 's1', () => true); await settled(); - sinon.assert.calledWithExactly(publish, 'syncthing:localChangesChecked', { folder: 'fluxa_a', onlyCopy: false }); + sinon.assert.calledWithExactly(publish, 'syncthing:localChangesChecked', { folder: 'fluxa_a', onlyCopy: false, madeHere: false }); } finally { publish.restore(); } }); }); - describe('findOnlyCopy', () => { + describe('examineLocalChanges', () => { it('asks about the items another device wrote, at most eight at once, and stops at the first only copy', async () => { const files = Array.from({ length: 20 }, (_, k) => ({ name: `f${k}`, version: ['PRIMARY:1'] })); syncthingServiceMock.getDbLocalChanged.resolves({ files }); syncthingServiceMock.getDbFile.withArgs('fluxa_a', 'f9').resolves({ global: { localFlags: 8 } }); - expect(await localChangeCheck.findOnlyCopy('fluxa_a')).to.equal('f9'); + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: 'f9', madeHere: false }); expect(syncthingServiceMock.getDbFile.callCount, 'asked past the batch that held it').to.equal(16); }); - it('asks nothing about an item only this node has written', async () => { + it('says a change was made here, asking nothing about it', async () => { syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); - expect(await localChangeCheck.findOnlyCopy('fluxa_a')).to.equal(false); + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: false, madeHere: true }); sinon.assert.notCalled(syncthingServiceMock.getDbFile); }); + it('says a change was made here when it is on a later page than the changes others hold', async () => { + const held = Array.from({ length: 1000 }, (_, k) => ({ name: `f${k}`, version: ['PRIMARY:1'] })); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a', 1).resolves({ files: held }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a', 2).resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); + + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: false, madeHere: true }); + }); + + it('says nothing was made here when every change names another device', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: false, madeHere: false }); + }); + it('cannot say when an item cannot be read', async () => { syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1'] }] }); syncthingServiceMock.getDbFile.rejects(new Error('404 No such object in the index')); - expect(await localChangeCheck.findOnlyCopy('fluxa_a')).to.equal(null); + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.equal(null); }); }); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 32de669143..3d69581ede 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -957,6 +957,8 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { + // The primary runs the app on another holder. + peerComponentMock.componentStateOnOtherHolders.resolves('running'); // The first pass asks for the check; the next acts on its answer. await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); @@ -987,6 +989,8 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { + // The primary runs the app on another holder. + peerComponentMock.componentStateOnOtherHolders.resolves('running'); // The first pass asks for the check; the next acts on its answer. await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); @@ -2413,7 +2417,7 @@ describe('syncthingFolderStateMachine tests', () => { expect(result.cache.restarted, 'a change only this node holds kept the folder from promoting').to.be.true; }); - it('is reverted when this node made it, asking nothing about it', async () => { + it('is reverted on an r: folder when this node made it, asking nothing about it', async () => { pollutedCandidate(); changes({ name: 'appdata/made-here', version: ['LOCAL-D:9'] }); @@ -2534,6 +2538,109 @@ describe('syncthingFolderStateMachine tests', () => { expect(result.cache.restarted).to.be.false; }); + // A change only this node made, on a single-writer folder, is the app's data + // while no other holder runs the app - its own writes as an earlier primary + // included - and junk beside a holder that runs it. + describe('made on this node, on a single-writer folder', () => { + const MADE_HERE = ['LOCAL-D:9']; + const othersRun = (state) => peerComponentMock.componentStateOnOtherHolders.resolves(state); + + it('is reverted while another holder runs the app', async () => { + readyStandby(); + changes({ name: 'appdata/stray', version: MADE_HERE }); + othersRun('running'); + + await checkedPass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + + it('is kept while no other holder runs the app, and said once', async () => { + readyStandby(); + changes({ name: 'appdata/written-as-primary', version: MADE_HERE }, { name: 'appdata', version: MADE_HERE }); + othersRun('notRunning'); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + await checkedPass(); + await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.calledOnceWithExactly( + publish.withArgs('syncthing:localChangesKept'), + 'syncthing:localChangesKept', + { + folder: 'test-app', files: 0, directories: 1, reason: 'noHolderRunsTheApp', + }, + ); + } finally { + publish.restore(); + } + }); + + it('is neither reverted nor promoted while whether another holder runs the app cannot be said', async () => { + pollutedCandidate(); + mockParams.containerDataFlags = 'g'; + changes({ name: 'appdata/stray', version: MADE_HERE }); + othersRun('unknown'); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + }); + + // The sole survivor of an app it was once the primary of: its folder becomes + // ready to take over with its changes, rather than being held for a revert + // that deletes them. + it('lets a folder become ready to take over while no other holder runs the app', async () => { + pollutedCandidate(); + mockParams.containerDataFlags = 'g'; + changes({ name: 'appdata/written-as-primary', version: MADE_HERE }); + othersRun('notRunning'); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.true; + }); + + it('is reverted once a holder runs the app, though it was kept while none did', async () => { + readyStandby(); + changes({ name: 'appdata/stray', version: MADE_HERE }); + othersRun('notRunning'); + await checkedPass(); + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + + othersRun('running'); + await pass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + + it('is kept beside a holder that runs the app when another change is the only copy of another device\'s data', async () => { + readyStandby(); + changes({ name: 'appdata/stray', version: MADE_HERE }, { name: 'appdata/only-here', version: FROM_PRIMARY }); + onlyHere('appdata/only-here'); + othersRun('running'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.notCalled(peerComponentMock.componentStateOnOtherHolders); + }); + + it('does not ask whether another holder runs the app when every change names another device', async () => { + readyStandby(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + + await checkedPass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + sinon.assert.notCalled(peerComponentMock.componentStateOnOtherHolders); + }); + }); + it('is kept on a ready standby, said once, and costs nothing on the passes after', async () => { readyStandby(); changes({ name: 'appdata/only-here', version: FROM_PRIMARY }); @@ -2548,7 +2655,7 @@ describe('syncthingFolderStateMachine tests', () => { sinon.assert.calledOnceWithExactly( publish.withArgs('syncthing:localChangesKept'), 'syncthing:localChangesKept', - { folder: 'test-app', files: 0, directories: 1 }, + { folder: 'test-app', files: 0, directories: 1, reason: 'onlyCopy' }, ); sinon.assert.calledOnce(syncthingServiceMock.getDbLocalChanged); sinon.assert.calledOnce(syncthingServiceMock.getDbFile); @@ -2804,6 +2911,8 @@ describe('syncthingFolderStateMachine tests', () => { globalBytes: 4096, globalFiles: 1, inSyncBytes: 4096, state: 'idle', receiveOnlyChangedFiles: 1, }); + // The primary runs the app on another holder. + peerComponentMock.componentStateOnOtherHolders.resolves('running'); // The first pass asks for the check; the next acts on its answer. await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); From 92abea8888bf395c2c012ad5ca4b7f2f2abf8ca6 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 21:59:52 +0100 Subject: [PATCH 301/369] test(harness): 3314 and 3315 revert a standby's own changes only while another holder runs the app Co-Authored-By: Claude Opus 5.5 --- ...y-holder-keeps-what-it-wrote-as-primary.js | 147 ++++++++++++++++++ ...-ran-the-app-are-reverted-once-one-does.js | 99 ++++++++++++ 2 files changed, 246 insertions(+) create mode 100644 test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js create mode 100644 test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js diff --git a/test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js b/test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js new file mode 100644 index 0000000000..5299172ca0 --- /dev/null +++ b/test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js @@ -0,0 +1,147 @@ +// weight: heavy +/* + * A node that was a single-writer app's primary, and later its standby, is left the + * only holder. What it wrote as the primary carries no other device's history, which + * is also what a standby's own junk carries; but no other holder runs the app, so + * this node holds its data, and a revert has nothing to restore it from. Its own + * earlier file and directory change on its disk: it keeps both and takes the app + * over with them. + * + * Reverted, the file is deleted - no other device holds it - and the directory, not + * empty, can never be: the folder is reverted on every pass and never taken over. + * + * A = the standby, then the primary; B = the primary, then the standby; C = holds + * nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { + crashFluxos, releaseFluxos, execInContainer, stopAppContainer, +} from '../framework/container.js'; +import { + getDeviceId, getFolderConfig, getFileInfo, readPath, scanFolder, startDaemon, stopDaemon, waitForDaemonEvent, folderSaved, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { clearMaster, electMaster } from '../framework/fdm-control.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// A machine that dies closes none of its connections, so a holder's syncthing keeps +// them until its own receive timeout (5 min). Then a few election passes and a start. +const TAKEOVER_MS = 10 * 60 * 1000; +// syncthing's FlagLocalReceiveOnly +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const FILE = 'appdata/written-as-primary.txt'; +const DIR = 'appdata/primary-dir'; +const CONTENT = 'written by B as the primary'; + +function step(what) { + console.log(`# 3314 ${new Date().toISOString()} ${what}`); +} + +describe('a former primary left the only holder keeps what it wrote as the primary', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2eformerprimary${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + // eslint-disable-next-line no-bitwise + const heldOnlyHere = (record) => ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; + const historyOf = async (i, item) => (await getFileInfo(client(i), folder, item))?.local?.version ?? []; + + // The machine of node `dead` dies, and `holder` drops it from the folder. + async function machineDies(dead, holder) { + const deadDevice = await getDeviceId(client(dead)); + await crashFluxos(client(dead).container, { hold: true }); + await Promise.all([A, B, C].filter((i) => i !== dead) + .map((i) => dbClient(i + 1).expireAppLocation(appName, client(dead).ip))); + await stopAppContainer(client(dead).container, appName, appName); + await stopDaemon(client(dead)); + await waitFor(async () => !((await getFolderConfig(client(holder), folder))?.devices ?? []) + .some((d) => d.deviceID === deadDevice), { + timeout: TAKEOVER_MS, interval: 5000, label: `node ${holder} drops node ${dead} from the folder`, + }); + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + await sh(B, `mkdir -p ${root}/${DIR} && printf '${CONTENT}' > ${root}/${FILE} && printf 'inside' > ${root}/${DIR}/inside.txt`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), `${root}/${FILE}`)) === CONTENT, { + timeout: 240000, interval: 3000, label: 'what the primary wrote reached the standby', + }); + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('hands the app to the standby when the primary dies', async function () { + this.timeout(TAKEOVER_MS + 300000); + step('B, the primary, dies'); + await machineDies(B, A); + await clearMaster(appName); + await waitFor(() => fleet.isUp(A), { timeout: TAKEOVER_MS, interval: 5000, label: 'A takes the app over' }); + await electMaster(appName, client(A).ip); + }); + + it('brings B back as a standby holding what it wrote as the primary', async function () { + this.timeout(600000); + step('B returns'); + await startDaemon(client(B), { paused: true }); + await releaseFluxos(client(B).container); + await waitForDaemonEvent(client(B), folderSaved(folder, { type: 'receiveonly', paused: false }), { + timeout: 300000, label: 'B\'s folder receiving', + }); + expect(await readPath(client(B), `${root}/${FILE}`), 'B holds what it wrote').to.equal(CONTENT); + const ownShortId = (await getDeviceId(client(B))).substring(0, 7); + expect((await historyOf(B, FILE)).every((counter) => counter.startsWith(`${ownShortId}:`)), + 'fixture: the file\'s history names a device other than B, so this is not B\'s own data').to.equal(true); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); + + it('keeps B\'s changes to what it wrote as the primary once it is the only holder, and B takes the app over with them', async function () { + this.timeout(TAKEOVER_MS * 2); + step('A, now the primary, dies'); + await machineDies(A, B); + + step('B\'s copy of its own file and directory changes on its disk'); + const changedAt = client(B).getLastEventId(); + await sh(B, `chmod 600 ${root}/${FILE} && chmod 700 ${root}/${DIR}`); + await scanFolder(client(B), folder); + await waitFor(async () => heldOnlyHere(await getFileInfo(client(B), folder, FILE)), { + timeout: 120000, interval: 2000, label: 'fixture: B\'s change is the only copy', + }); + + // Whichever B decides first: kept or reverted. + const decisions = () => client(B).getEventBuffer().filter((e) => e.id > changedAt && e.data?.folder === folder + && (e.event === 'syncthing:localChangesKept' || e.event === 'syncthing:localChangesReverted')); + await waitFor(async () => decisions().length > 0, { + timeout: 300000, interval: 2000, label: 'B decides what to do with its changes', + }); + expect(decisions()[0].event, 'B reverted the only copy of what it wrote as the primary').to.equal('syncthing:localChangesKept'); + expect(decisions()[0].data.reason, 'why B kept its changes').to.equal('noHolderRunsTheApp'); + expect(await readPath(client(B), `${root}/${FILE}`), 'B\'s file is gone').to.equal(CONTENT); + + step('B takes the app over'); + await electMaster(appName, client(B).ip); + await waitFor(() => fleet.isUp(B), { timeout: TAKEOVER_MS, interval: 5000, label: 'B takes the app over' }); + expect(await readPath(client(B), `${root}/${FILE}`), 'B took the app over without its file').to.equal(CONTENT); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js b/test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js new file mode 100644 index 0000000000..1e33dc18fe --- /dev/null +++ b/test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js @@ -0,0 +1,99 @@ +// weight: heavy +/* + * A standby's own changes are reverted only while another holder runs the app. The + * primary's machine dies, and a change the standby made itself is kept: no holder + * runs the app, so nothing says it is junk. The primary comes back and runs the app + * again, and the standby, asking again on every pass, reverts the change: the + * primary is authoritative and does not have it. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { + crashFluxos, releaseFluxos, execInContainer, stopAppContainer, +} from '../framework/container.js'; +import { + getConnectedDevices, getDeviceId, readPath, scanFolder, startDaemon, stopDaemon, +} from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// A machine that dies closes none of its connections, so the standby's syncthing +// holds them until its own receive timeout (5 min). +const DISCONNECT_MS = 10 * 60 * 1000; +const PASSES_AFTER_RETURN = 3; +const STRAY = 'appdata/made-on-the-standby.txt'; + +function step(what) { + console.log(`# 3315 ${new Date().toISOString()} ${what}`); +} + +describe('a standby\'s own changes kept while no holder ran the app are reverted once one does', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2ekeptthenreverted${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const passes = () => client(A).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + const eventsSince = (name, from) => client(A).getEventBuffer().filter((e) => e.id > from && e.event === name && e.data?.folder === folder); + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('keeps a change the standby made while no holder runs the app', async function () { + this.timeout(DISCONNECT_MS + 300000); + step('the primary\'s machine dies; FDM still names it'); + const primaryDevice = await getDeviceId(client(B)); + await crashFluxos(client(B).container, { hold: true }); + await stopAppContainer(client(B).container, appName, appName); + await stopDaemon(client(B)); + await waitFor(async () => !(await getConnectedDevices(client(A))).includes(primaryDevice), { + timeout: DISCONNECT_MS, interval: 5000, label: 'the standby\'s syncthing loses the primary', + }); + + step('the standby makes a change of its own'); + const from = client(A).getLastEventId(); + const wrote = await execInContainer(client(A).container, `printf 'standby' > ${root}/${STRAY}`); + expect(wrote.exitCode, `fixture: ${wrote.output}`).to.equal(0); + await scanFolder(client(A), folder); + const kept = await client(A).waitForEvent('syncthing:localChangesKept', (d) => d.folder === folder, 300000, { afterId: from }); + expect(kept.data.reason, 'why the standby kept its change').to.equal('noHolderRunsTheApp'); + expect(eventsSince('syncthing:localChangesReverted', from), 'the standby reverted while no holder ran the app').to.deep.equal([]); + expect(await readPath(client(A), `${root}/${STRAY}`), 'the standby\'s change').to.equal('standby'); + }); + + it('reverts that change once the primary runs the app again', async function () { + this.timeout(900000); + step('the primary comes back and runs the app'); + const from = client(A).getLastEventId(); + await startDaemon(client(B), { paused: true }); + await releaseFluxos(client(B).container); + await waitFor(() => fleet.isUp(B), { timeout: 600000, interval: 5000, label: 'the primary runs the app again' }); + + const passesWithPrimaryUp = await passes(); + await waitFor(async () => await passes() >= passesWithPrimaryUp + PASSES_AFTER_RETURN, { + timeout: 180000, interval: 2000, label: `${PASSES_AFTER_RETURN} of the standby's passes with the primary running`, + }); + expect(eventsSince('syncthing:localChangesReverted', from).length, 'the standby kept its change beside a primary running the app').to.be.at.least(1); + await waitFor(async () => (await readPath(client(A), `${root}/${STRAY}`)) === null, { + timeout: 120000, interval: 2000, label: 'the standby\'s change is gone', + }); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); +}); From ac454fca69a5c0f879850e453691c0ce010b0646 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:41:38 +0100 Subject: [PATCH 302/369] fix(syncthing): a holder is in an app's folders while a location lists it, and only then A node listed at an address that gives no device - its FluxOS does not answer, or its syncthing has not answered it - keeps the device syncthing has configured under that address, counted as `configured`, so a listed holder stays in its folders across this node's restarts. A device no location lists leaves the folder whether or not it is still connected: a node whose FluxOS has stopped answering stops announcing while its syncthing and its app carry on, and once its location expires its writes reach no holder through this one. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/syncthingMonitor.js | 43 ------------ .../appMonitoring/syncthingMonitorHelpers.js | 24 +++++-- tests/unit/syncthingMonitor.test.js | 16 ++--- tests/unit/syncthingMonitorHelpers.test.js | 67 +++++++++++++++++++ 4 files changed, 89 insertions(+), 61 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index dfb588dc4c..77549f2b52 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -256,35 +256,6 @@ async function readAppLocation(appName) { return results || []; } -// `|` for each device connectedUnnamedDevices keeps, so keeping -// it is said once. -const keptUnnamedDevices = new Set(); - -/** - * The devices of a folder that its app's locations no longer name but that are - * still connected and sharing it. - * - * A location expires when its node stops announcing, and a node can stop - * announcing while its syncthing carries on - a FluxOS that has stopped answering - * leaves the app and syncthing running. Such a holder may still be the app's - * primary, writing, so it keeps its place until its connection closes or it stops - * sharing the folder. A device whose state cannot be read keeps its place too. - * @param {string} folderId - * @param {Array<{deviceID: string}>} configured The folder's devices in syncthing now. - * @param {Array<{deviceID: string}>} named The devices the app's locations name. - * @returns {Promise} Device IDs to keep in the folder. - */ -async function connectedUnnamedDevices(folderId, configured, named) { - const unnamed = (configured || []) - .map((device) => device.deviceID) - .filter((deviceID) => !named.some((device) => device.deviceID === deviceID)); - const kept = await Promise.all(unnamed.map(async (deviceID) => { - const completion = await syncthingService.getDbCompletion({ folder: folderId, device: deviceID }).catch(() => null); - return !completion || completion.remoteState === 'valid' ? deviceID : null; - })); - return kept.filter(Boolean); -} - /** * An app's location records, or none when they cannot be read. * @param {string} appName @@ -395,20 +366,6 @@ async function processContainerData(params) { allDevices, ); - if (syncFolder) { - const kept = await connectedUnnamedDevices(id, syncFolder.devices, devices); - [...keptUnnamedDevices].filter((key) => key.startsWith(`${id}|`) && !kept.includes(key.slice(id.length + 1))) - .forEach((key) => keptUnnamedDevices.delete(key)); - kept.forEach((deviceID) => { - if (!keptUnnamedDevices.has(`${id}|${deviceID}`)) { - keptUnnamedDevices.add(`${id}|${deviceID}`); - log.info(`processContainerData - ${appId}: keeping device ${deviceID.substring(0, 7)}, connected and sharing the folder though no location names it`); - } - devices.push({ deviceID }); - if (!devicesIds.includes(deviceID)) devicesIds.push(deviceID); - }); - } - // Create base folder configuration const syncthingFolder = createSyncthingFolderConfig(id, label, folder, devices); diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js b/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js index b5697cb837..0f726d5c34 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js @@ -56,15 +56,21 @@ async function getDeviceID(fluxIP, retries = 0) { * folder syncing with a device that holds no copy of it. A peer that cannot * prove either way is asked the unsigned way. * + * A node listed at `name` that gives no device - its FluxOS does not answer, or + * its syncthing has not answered it - keeps the device syncthing has configured + * under `name`, so a holder stays in its folders for as long as it is listed. + * * Every call counts where its answer came from under `syncthing:deviceSource`, - * keyed by `name`: verified, verifiedNoDevice, introduced, withheld, held or - * unsigned. + * keyed by `name`: verified, verifiedNoDevice, introduced, withheld, held, + * unsigned or configured. * * @param {string} name - Device name (IP:port) * @param {Map} cache - Cache map + * @param {string|null} [configured] - The device syncthing has configured under + * `name`, when exactly one is * @returns {Promise} Device ID or null */ -async function getDeviceIDCached(name, cache) { +async function getDeviceIDCached(name, cache, configured = null) { const { IdentityVerdict } = peerIdentityService; const from = (source, deviceID) => { fluxEventBus.count('syncthing:deviceSource', name, source); @@ -76,7 +82,10 @@ async function getDeviceIDCached(name, cache) { const signedDeviceID = result.identity.deviceId; // Verified with no device: that node's syncthing has not answered it yet. // What was held stands until it has. - if (!signedDeviceID) return from('verifiedNoDevice', cache.get(name) ?? null); + if (!signedDeviceID) { + if (cache.has(name)) return from('verifiedNoDevice', cache.get(name)); + return configured ? from('configured', configured) : from('verifiedNoDevice', null); + } cache.set(name, signedDeviceID); return from('verified', signedDeviceID); } @@ -100,8 +109,9 @@ async function getDeviceIDCached(name, cache) { const deviceID = await getDeviceID(name); if (deviceID) { cache.set(name, deviceID); + return from('unsigned', deviceID); } - return from('unsigned', deviceID); + return configured ? from('configured', configured) : from('unsigned', null); } /** @@ -184,7 +194,9 @@ async function buildDeviceConfiguration( const addresses = peerSyncthingAddresses(ip, port); const name = `${ip}:${port}`; - const deviceID = await getDeviceIDCached(name, deviceCache); + const underName = allDevices.filter((device) => device.name === name); + const configured = underName.length === 1 ? underName[0].deviceID : null; + const deviceID = await getDeviceIDCached(name, deviceCache, configured); if (!deviceID) { return null; diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index b55670700d..e6e69c8b7a 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -1933,8 +1933,6 @@ describe('syncthingMonitor tests', () => { // because a pass failed to reach the component that owns it. Deleting a live // folder drops syncthing's whole record of the app: the index, the peer // devices and, on the next pass, any mount-safety demotion standing on it. - // A location expires when its node stops announcing, which a node whose FluxOS has - // stopped answering does while its syncthing, and the app, carry on. describe('a peer no location names any more', () => { function folderSharedWithPeer() { mockState.backupInProgress = []; @@ -1960,22 +1958,16 @@ describe('syncthingMonitor tests', () => { syncthingServiceMock.getDeviceStats.resolves({}); }); - it('keeps its device while it is connected and shares the folder', async () => { + // A node whose FluxOS has stopped answering stops announcing while its + // syncthing and its app carry on; once its location expires its writes reach + // no holder through this one. + it('removes its device though it is connected and shares the folder', async () => { folderSharedWithPeer(); syncthingServiceMock.getDbCompletion.withArgs({ folder: 'testapp', device: 'PEER-DEVICE' }).resolves({ remoteState: 'valid' }); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); - sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); - }); - - it('removes its device once it no longer shares the folder', async () => { - folderSharedWithPeer(); - - monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); - await clock.tickAsync(100); - sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); }); diff --git a/tests/unit/syncthingMonitorHelpers.test.js b/tests/unit/syncthingMonitorHelpers.test.js index 37552f2639..ffe40c35c1 100644 --- a/tests/unit/syncthingMonitorHelpers.test.js +++ b/tests/unit/syncthingMonitorHelpers.test.js @@ -404,6 +404,41 @@ describe('syncthingMonitorHelpers tests', () => { expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map())).to.equal(null); }); + // A listed holder whose FluxOS has stopped answering, asked by a node whose + // FluxOS restarted and so holds nothing for it. + it('keeps the device syncthing has configured under the address of a listed peer that does not answer', async () => { + const cache = new Map(); + sandbox.stub(axios, 'get').rejects(new Error('connect ETIMEDOUT')); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', cache, 'CONFIGURED-DEVICE')).to.equal('CONFIGURED-DEVICE'); + expect(cache.has('10.0.0.1:16127'), 'a configured device is asked again on the next pass').to.equal(false); + counted('configured'); + }); + + it('keeps the configured device while a verified peer\'s syncthing has not answered and nothing is held', async () => { + verifyPeer.resolves({ + verdict: IdentityVerdict.VERIFIED, + identity: { socketAddress: '10.0.0.1:16127', pubKey: 'PUB', deviceId: null }, + }); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map(), 'CONFIGURED-DEVICE')).to.equal('CONFIGURED-DEVICE'); + counted('configured'); + }); + + it('takes a peer\'s unsigned answer over the device configured under its address', async () => { + sandbox.stub(axios, 'get').resolves({ data: { status: 'success', data: 'ANSWERED-DEVICE' } }); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map(), 'CONFIGURED-DEVICE')).to.equal('ANSWERED-DEVICE'); + counted('unsigned'); + }); + + it('configures no device for an address another node answered, whatever is configured under it', async () => { + verifyPeer.resolves({ verdict: IdentityVerdict.MISROUTED, answeredAs: '10.0.0.9:16147' }); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map(), 'CONFIGURED-DEVICE')).to.equal(null); + counted('withheld'); + }); + // The palworld case: HK held its neighbour's device under UAE's name for five // days, and the folder never synced. it('configures no device for an address another node answered, and forgets the one it held', async () => { @@ -487,6 +522,38 @@ describe('syncthingMonitorHelpers tests', () => { expect(devicesIds).to.deep.equal(['REAL-DEVICE']); }); + describe('a listed peer that gives no device', () => { + beforeEach(() => { + peerIdentityService.verifyPeer.resolves({ verdict: IdentityVerdict.UNVERIFIABLE, reason: 'connect ETIMEDOUT' }); + sandbox.stub(axios, 'get').rejects(new Error('connect ETIMEDOUT')); + }); + + it('stays in the folder on the device configured under its address', async () => { + const devicesIds = []; + const allDevices = [{ name: '10.0.0.1:16127', deviceID: 'CONFIGURED-DEVICE' }]; + + const devices = await helpers.buildDeviceConfiguration( + [{ ip: '10.0.0.1:16127' }], LOCAL, 'MY-DEVICE', new Map(), [], devicesIds, allDevices, + ); + + expect(devices).to.deep.equal([{ deviceID: 'MY-DEVICE' }, { deviceID: 'CONFIGURED-DEVICE' }]); + expect(devicesIds, 'the sweep of unused devices would remove it').to.deep.equal(['CONFIGURED-DEVICE']); + }); + + it('leaves the folder while more than one device is configured under its address', async () => { + const allDevices = [ + { name: '10.0.0.1:16127', deviceID: 'CONFIGURED-DEVICE' }, + { name: '10.0.0.1:16127', deviceID: 'DEVICE-OF-A-NEIGHBOUR' }, + ]; + + const devices = await helpers.buildDeviceConfiguration( + [{ ip: '10.0.0.1:16127' }], LOCAL, 'MY-DEVICE', new Map(), [], [], allDevices, + ); + + expect(devices).to.deep.equal([{ deviceID: 'MY-DEVICE' }]); + }); + }); + it('configures a new peer device not to auto-accept folders', async () => { const devicesConfiguration = []; From 9134a2e168943dcd7df5a3b7799aba87d003204b Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:51:52 +0100 Subject: [PATCH 303/369] fix(syncthing): a primary no location lists is out of the app, and a silent peer's liveness is its connection, whatever folders it shares A remembered primary that no location lists is not asked and holds no component: its location expired, every holder has dropped it from the app's folders, and what it writes reaches none of them. The election counts it as `previousPrimaryUnlisted` and goes on to choose the next primary. A silent peer is connected while this node's syncthing holds a live connection to its device (`getConnectedDeviceIds`), not only while it shares a given folder: a holder dropped from a folder stays connected, and syncthing reports a connected device's last-seen as now. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 13 +- .../services/appMonitoring/peerComponent.js | 10 +- .../appMonitoring/peerFolderLiveness.js | 29 ++--- .../syncthingFolderStateMachine.js | 4 +- ZelBack/src/services/syncthingService.js | 14 +- tests/unit/advancedWorkflows.test.js | 123 +++++++++++++----- tests/unit/peerComponent.test.js | 29 ++--- .../unit/syncthingFolderStateMachine.test.js | 16 +-- 8 files changed, 150 insertions(+), 88 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 9854e662a5..c5f24f1c26 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -5404,8 +5404,17 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // primary this node cannot read keeps the component. It is the // instance most likely to still hold the volume, and electing over // it is exactly the split-brain this branch is reached to avoid. - // eslint-disable-next-line no-await-in-loop - const previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); + // A previous primary no location lists is out of the app and is not + // asked: its location expired, every holder has dropped it from the + // app's folders, and what it writes reaches none of them. + let previousMasterState = PeerComponent.NOT_RUNNING; + if (previousMasterNode) { + // eslint-disable-next-line no-await-in-loop + previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); + } else { + fluxEventBus.count('masterSlave:decision', identifier, 'previousPrimaryUnlisted'); + log.info(`masterSlaveApps: app:${installedApp.name} previous primary ${previousMasterAddr} holds no location - it is out of the app`); + } if (previousMasterState !== PeerComponent.NOT_RUNNING) { if (previousMasterState === PeerComponent.RUNNING) withdrawSeedMarks(members, 'the previous primary runs it'); if (resuming && previousMasterState === PeerComponent.RUNNING) { diff --git a/ZelBack/src/services/appMonitoring/peerComponent.js b/ZelBack/src/services/appMonitoring/peerComponent.js index cd9d571bb9..e8505d72f9 100644 --- a/ZelBack/src/services/appMonitoring/peerComponent.js +++ b/ZelBack/src/services/appMonitoring/peerComponent.js @@ -192,21 +192,19 @@ async function peerComponentState(peerSocketAddr, { /** * What a peer that did not reply at all can be shown to be doing with an app's * g: components: free only when this node's own syncthing shows the peer's - * connection gone for every one of their folders. A folder still connected is a - * component the peer may still be writing. + * connection gone. A peer still connected may still be writing them. * @param {string} peerSocketAddr The peer's socket address. * @param {object} ctx As peerComponentState. * @returns {Promise} NOT_RUNNING or UNKNOWN. */ async function silentPeerState(peerSocketAddr, { - appIds, identifier, appName, liveness, label, logPrefix, + identifier, appName, liveness, label, logPrefix, }) { const ipToCheck = extractIp(peerSocketAddr); - const verdicts = await Promise.all(appIds.map((appId) => silenceVerdict(appId, peerSocketAddr, liveness))); - const verdict = verdicts.find((each) => each !== SilenceVerdict.GONE) ?? SilenceVerdict.GONE; + const verdict = await silenceVerdict(peerSocketAddr, liveness); fluxEventBus.count('peer:silenceVerdict', identifier, verdict); if (verdict === SilenceVerdict.GONE) { - log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appIds.join(', ')} gone - the app is free there`); + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection gone - the app is free there`); return PeerComponent.NOT_RUNNING; } log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent for app:${appName} and ${SILENCE_REASONS[verdict]}, will not start`); diff --git a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js index 48fb81a8fa..bb2490fd16 100644 --- a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js +++ b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js @@ -349,15 +349,15 @@ async function peerDeviceId(peerIp) { } /** - * This node's syncthing's view of its own connection to a peer, for one - * folder: PeerConnection.CONNECTED, DISCONNECTED, or UNKNOWN. + * This node's syncthing's view of its own connection to a peer: + * PeerConnection.CONNECTED, DISCONNECTED, or UNKNOWN. * * FluxOS and syncthing are separate processes on that peer and fail * independently, so silence from its API is not evidence that it has stopped * writing - a FluxOS restart takes the API away for tens of seconds while - * syncthing and the container carry on. A live sync connection IS evidence: - * syncthing reports remoteState 'valid' only for a device whose connection is - * open, and clears it the moment the connection closes. + * syncthing and the container carry on. A live sync connection IS evidence, + * whatever folders it shares: a holder no location lists any more is dropped + * from the app's folders while its syncthing stays connected to this one. * * The three answers are kept apart because a silence may be acted on only with * evidence. 'disconnected' is an answer - this node's syncthing was asked @@ -380,22 +380,16 @@ async function peerDeviceId(peerIp) { * does not need to be. This is local state, maintained by the connection * itself. * - * @param {string} folderId Folder id, always passed explicitly: the completion - * endpoint's aggregate form never sets remoteState and reports 'unknown'. * @param {string} peerIp * @returns {Promise} One of PeerConnection */ -async function peerSyncthingConnection(folderId, peerIp) { +async function peerSyncthingConnection(peerIp) { const deviceId = await peerDeviceId(peerIp); if (!deviceId) return PeerConnection.UNKNOWN; - const completion = await syncthingService.getDbCompletion({ - folder: folderId, - device: deviceId, - }).catch(() => null); - - if (!completion) return PeerConnection.UNKNOWN; - if (completion.remoteState === 'valid') return PeerConnection.CONNECTED; + const connected = await syncthingService.getConnectedDeviceIds().catch(() => null); + if (!connected) return PeerConnection.UNKNOWN; + if (connected.has(deviceId)) return PeerConnection.CONNECTED; // A device this node's syncthing has paused is disconnected by this node, not // by the peer; a config that cannot be read cannot rule that out. @@ -446,13 +440,12 @@ async function peerSyncthingConnection(folderId, peerIp) { * time keeps what it holds. Once it has passed, the closed connection is * evidence again. * - * @param {string} folderId * @param {string} peerIp * @param {Object} liveness This pass's peer view * @returns {Promise} One of SilenceVerdict */ -async function silenceVerdict(folderId, peerIp, liveness) { - const connection = await peerSyncthingConnection(folderId, peerIp); +async function silenceVerdict(peerIp, liveness) { + const connection = await peerSyncthingConnection(peerIp); if (connection === PeerConnection.CONNECTED) return SilenceVerdict.CONNECTION_ALIVE; if (connection === PeerConnection.UNKNOWN) return SilenceVerdict.NO_EVIDENCE; const shutdown = await announcedShutdown(peerIp); diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 33bf5f8e26..897c4de0ee 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -798,9 +798,9 @@ async function holderIsGone(appId, holderIp, liveness) { const answer = await liveness.read(holderIp); if (answer.reachable) return false; - const verdict = await silenceVerdict(appId, holderIp, liveness); + const verdict = await silenceVerdict(holderIp, liveness); if (verdict === SilenceVerdict.CONNECTION_ALIVE) { - log.info(`holderIsGone - ${extractIp(holderIp)}'s API is silent, but this node's syncthing still holds a live connection to it for ${appId}; it is restarting, not gone`); + log.info(`holderIsGone - ${extractIp(holderIp)}'s API is silent, but this node's syncthing still holds a live connection to it; ${appId}'s holder is restarting, not gone`); fluxEventBus.publish('syncthing:holderRetained', { folder: appId, holder: holderIp, reason: 'connectionAlive' }); return false; } diff --git a/ZelBack/src/services/syncthingService.js b/ZelBack/src/services/syncthingService.js index 0e81be9fba..097843d6e0 100644 --- a/ZelBack/src/services/syncthingService.js +++ b/ZelBack/src/services/syncthingService.js @@ -943,6 +943,16 @@ async function folderCompleteOnPeers(folder, myId, connectedPeers) { return completions.every((completion) => completion.needBytes === 0 && completion.needItems === 0 && completion.needDeletes === 0); } +/** + * The devices this node's syncthing holds a live connection to, whatever folders + * they share. + * @returns {Promise>} Device ids + */ +async function getConnectedDeviceIds() { + const { connections = {} } = await request('get', '/rest/system/connections'); + return new Set(Object.keys(connections).filter((id) => connections[id]?.connected)); +} + /** * Syncthing's statistics per device, keyed by device id. lastSeen is when the * device last connected or disconnected, to the second, and "now" while it is @@ -1021,8 +1031,7 @@ async function drainFoldersToPeers(timeoutMs) { const folders = (await getConfigFolders()).filter((folder) => folder.type === 'sendreceive'); if (!folders.length) return []; const myId = await getDeviceId(); - const { connections = {} } = await request('get', '/rest/system/connections'); - const connectedPeers = new Set(Object.keys(connections).filter((id) => connections[id]?.connected)); + const connectedPeers = await getConnectedDeviceIds(); const scans = await Promise.allSettled(folders.map((folder) => scanFolder(folder.id, { timeoutMs: Math.max(1, deadline - monotonicMs()), }))); @@ -2753,6 +2762,7 @@ module.exports = { eachDbLocalChanged, dbRevert, drainFoldersToPeers, + getConnectedDeviceIds, getDeviceStats, scanFolder, // EVENTS diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 6c3abf18aa..1006e43e88 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -803,7 +803,7 @@ describe('advancedWorkflows tests', () => { let shuttingDownNodesStub; let dockerServiceStub; let syncthingServiceStub; - let syncthingCompletionStub; + let syncthingConnectedStub; let syncthingDeviceStatsStub; let syncthingStatusStub; let syncthingDevicesStub; @@ -895,7 +895,8 @@ describe('advancedWorkflows tests', () => { // (no device ever resolved) on a healthy fleet, so a test that wants a // silence acted on has to supply the evidence for it. const syncthingServiceModule = require('../../ZelBack/src/services/syncthingService'); - syncthingCompletionStub = sinon.stub(syncthingServiceModule, 'getDbCompletion').resolves(null); + sinon.stub(syncthingServiceModule, 'getDbCompletion').resolves(null); + syncthingConnectedStub = sinon.stub(syncthingServiceModule, 'getConnectedDeviceIds').resolves(new Set()); syncthingDevicesStub = sinon.stub(syncthingServiceModule, 'getConfigDevices').resolves([]); // default: this node's syncthing has never been connected to any peer syncthingDeviceStatsStub = sinon.stub(syncthingServiceModule, 'getDeviceStats').resolves({}); @@ -1159,13 +1160,13 @@ describe('advancedWorkflows tests', () => { }; // This node's own syncthing view of a peer's device, which is what a silence is - // judged on. 'valid' is a live connection; any other state is a closed one if - // the connection was ever up, and no answer if it never was; and leaving the + // judged on. 'connected' is a live connection; 'closed' is a closed one if the + // connection was ever up, and no answer if it never was; and leaving the // device out of the cache entirely is the third answer - this node never // resolved the peer and cannot say. - const peerSyncthingSays = (peerSocketAddr, remoteState, { everConnected = true } = {}) => { + const peerSyncthingSays = (peerSocketAddr, connection, { everConnected = true } = {}) => { globalState.syncthingDevicesIDCache.set(peerSocketAddr, `DEVICE-${peerSocketAddr}`); - syncthingCompletionStub.resolves({ remoteState }); + syncthingConnectedStub.resolves(new Set(connection === 'connected' ? [`DEVICE-${peerSocketAddr}`] : [])); syncthingDeviceStatsStub.resolves(everConnected ? { [`DEVICE-${peerSocketAddr}`]: { lastSeen: '2026-09-25T14:00:00Z' } } : { [`DEVICE-${peerSocketAddr}`]: { lastSeen: '1970-01-01T00:00:00Z' } }); @@ -1624,7 +1625,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('previous primary unreachable')); - peerSyncthingSays(PEER, 'valid'); + peerSyncthingSays(PEER, 'connected'); await runPass(); sinon.assert.notCalled(releaseOperatorStart); @@ -1644,7 +1645,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('peer unreachable')); - peerSyncthingSays(PEER, 'valid'); + peerSyncthingSays(PEER, 'connected'); await runPass(); @@ -2537,7 +2538,7 @@ describe('advancedWorkflows tests', () => { peerIdentityService.askSigned.withArgs('192.168.1.90:16127').resolves({ verdict: peerIdentityService.IdentityVerdict.UNREACHABLE, reason: 'connect ECONNREFUSED', }); - peerSyncthingSays('192.168.1.90:16127', 'valid'); + peerSyncthingSays('192.168.1.90:16127', 'connected'); await runPass(); @@ -2974,7 +2975,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); // the peer answers nothing at all (the axios default), but its sync connection // to this node is open - peerSyncthingSays('192.168.1.90:16127', 'valid'); + peerSyncthingSays('192.168.1.90:16127', 'connected'); await runPass(); @@ -3009,7 +3010,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); await runPass(); @@ -3025,7 +3026,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); shuttingDownNodesStub.resolves(['192.168.1.90:16127']); await runPass(); @@ -3040,7 +3041,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); shuttingDownNodesStub.resolves(['192.168.1.90:16137']); await runPass(); @@ -3054,7 +3055,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); shuttingDownNodesStub.rejects(new Error('database unavailable')); await runPass(); @@ -3078,7 +3079,6 @@ describe('advancedWorkflows tests', () => { // nothing cached, but this node's syncthing still has the device configured // under the name the monitor gave it, and reports the connection closed syncthingDevicesStub.resolves([{ name: '192.168.1.90:16127', deviceID: 'DEVICE-DEAD-PEER' }]); - syncthingCompletionStub.resolves({ remoteState: 'unknown' }); syncthingDeviceStatsStub.resolves({ 'DEVICE-DEAD-PEER': { lastSeen: '2026-09-25T14:00:00Z' } }); await runPass(); @@ -3096,7 +3096,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown', { everConnected: false }); + peerSyncthingSays('192.168.1.90:16127', 'closed', { everConnected: false }); await runPass(); @@ -3111,7 +3111,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingDevicesStub.resolves([{ name: '192.168.1.90:16127', deviceID: 'DEVICE-192.168.1.90:16127', paused: true }]); await runPass(); @@ -3127,7 +3127,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingStatusStub.resolves({ startTime: '2026-09-25T14:00:30Z' }); await runPass(); @@ -3143,7 +3143,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingStatusStub.resolves({ startTime: '2026-09-25T14:00:00Z' }); await runPass(); @@ -3158,7 +3158,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingStatusStub.resolves({ startTime: '2026-09-25T13:59:59Z' }); await runPass(); @@ -3182,7 +3182,7 @@ describe('advancedWorkflows tests', () => { sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); const runPass = electionFixture(appName, [PEER]); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays(PEER, 'unknown', options); + peerSyncthingSays(PEER, 'closed', options); syncthingStatusStub.resolves({ startTime }); return runPass; }; @@ -3253,7 +3253,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingStatusStub.rejects(new Error('syncthing did not answer')); await runPass(); @@ -3267,7 +3267,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingDevicesStub.rejects(new Error('syncthing did not answer')); await runPass(); @@ -3282,7 +3282,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); syncthingDeviceStatsStub.rejects(new Error('syncthing busy')); await runPass(); @@ -3300,7 +3300,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); const fluxCommunication = require('../../ZelBack/src/services/fluxCommunication'); fluxCommunication.peerResponsiveness.returns({ responding: 1, total: 8 }); @@ -3321,7 +3321,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(Object.assign(new Error('Request failed with status code 500'), { response: { status: 500 } })); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); await runPass(); @@ -3536,7 +3536,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(Object.assign(new Error('Request failed with status code 500'), { response: { status: 500 } })); - peerSyncthingSays('192.168.1.90:16127', 'unknown'); + peerSyncthingSays('192.168.1.90:16127', 'closed'); await runPass(); @@ -3835,7 +3835,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('previous primary unreachable')); - peerSyncthingSays('192.168.1.90:16127', 'valid'); + peerSyncthingSays('192.168.1.90:16127', 'connected'); await runPass(); @@ -3913,14 +3913,17 @@ describe('advancedWorkflows tests', () => { axiosGetStub.callsFake(peerAnswers({ held: [`flux${appName}`] })); await runPass(); - // Its location lapses and FDM names nobody; its address is silent, and this - // node's syncthing is connected to its device. + // FDM names nobody; its address is silent, and this node's syncthing is + // connected to its device. serviceHelperStub.resetBehavior(); serviceHelperStub.resolves(fdmNoPrimary()); - registryManagerStub.resolves([{ name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }]); + registryManagerStub.resolves([ + { name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }, + { name: appName, ip: PRIMARY, runningSince: '2026-01-01T00:00:00.000Z' }, + ]); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('silent')); - peerSyncthingSays(PRIMARY, 'valid'); + peerSyncthingSays(PRIMARY, 'connected'); logInfo.resetHistory(); await runPass(); @@ -3929,6 +3932,62 @@ describe('advancedWorkflows tests', () => { expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); + // Its location expired, so every holder has dropped it from the app's folders + // and what it writes reaches none of them: it is out of the app, whatever its + // syncthing still says. + it('takes a remembered primary no location lists as out of the app, without asking it', async () => { + const appName = 'unlistedprimary'; + const PRIMARY = '192.168.1.90:16127'; + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const logInfo = sinon.stub(log, 'info'); + const count = sinon.stub(fluxEventBus, 'count'); + const runPass = electionFixture(appName, [PRIMARY]); + serviceHelperStub.resetBehavior(); + serviceHelperStub.resolves(fdmNames('192.168.1.90')); + axiosGetStub.resetBehavior(); + axiosGetStub.callsFake(peerAnswers({ held: [`flux${appName}`] })); + await runPass(); + + serviceHelperStub.resetBehavior(); + serviceHelperStub.resolves(fdmNoPrimary()); + registryManagerStub.resolves([{ name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }]); + axiosGetStub.resetBehavior(); + axiosGetStub.rejects(new Error('silent')); + peerSyncthingSays(PRIMARY, 'connected'); + logInfo.resetHistory(); + await runPass(); + + expect(axiosGetStub.getCalls().some((call) => String(call.args[0]).includes(PRIMARY)), 'the unlisted primary was asked').to.equal(false); + sinon.assert.calledWith(count, 'masterSlave:decision', appName, 'previousPrimaryUnlisted'); + expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); + }); + + // Canary for the test above: listed, the same silent and connected primary + // keeps the app. + it('keeps the app for a remembered primary a location still lists, silent and connected', async () => { + const appName = 'listedprimary'; + const PRIMARY = '192.168.1.90:16127'; + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const logInfo = sinon.stub(log, 'info'); + const runPass = electionFixture(appName, [PRIMARY]); + serviceHelperStub.resetBehavior(); + serviceHelperStub.resolves(fdmNames('192.168.1.90')); + axiosGetStub.resetBehavior(); + axiosGetStub.callsFake(peerAnswers({ held: [`flux${appName}`] })); + await runPass(); + + serviceHelperStub.resetBehavior(); + serviceHelperStub.resolves(fdmNoPrimary()); + axiosGetStub.resetBehavior(); + axiosGetStub.rejects(new Error('silent')); + peerSyncthingSays(PRIMARY, 'connected'); + logInfo.resetHistory(); + await runPass(); + + expect(linesMatching(logInfo, 'still holds a live connection to it')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); + }); + it('reads FDM naming the address this node moved from as naming this node, and starts as its primary', async () => { const appName = 'movedself'; sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); diff --git a/tests/unit/peerComponent.test.js b/tests/unit/peerComponent.test.js index c2abcdd6bd..e7ac69146c 100644 --- a/tests/unit/peerComponent.test.js +++ b/tests/unit/peerComponent.test.js @@ -142,14 +142,13 @@ describe('peerComponent tests', () => { }); describe('a peer that does not answer', () => { - // This node's syncthing saw the peer after it started, and holds no live - // connection to it for whichever folders the test names. - const evidence = (connectedFolders) => { + // This node's syncthing saw the peer after it started, and holds a live + // connection to it or not, whatever folders it shares with it. + const evidence = ({ connected }) => { sinon.stub(axios, 'get').rejects(new Error('connect ECONNREFUSED')); sinon.stub(syncthingService, 'getConfigDevices').resolves([{ name: PEER, deviceID: 'PEERDEVICE' }]); - sinon.stub(syncthingService, 'getDbCompletion').callsFake(async ({ folder }) => ({ - remoteState: connectedFolders.includes(folder) ? 'valid' : 'notSharing', - })); + sinon.stub(syncthingService, 'getConnectedDeviceIds').resolves(new Set(connected ? ['PEERDEVICE'] : [])); + sinon.stub(syncthingService, 'getDbCompletion').resolves({ remoteState: 'notSharing' }); sinon.stub(syncthingService, 'getDeviceStats').resolves({ PEERDEVICE: { lastSeen: '2026-10-16T10:00:00Z' } }); sinon.stub(syncthingService, 'getSystemStatus').resolves({ startTime: '2026-10-16T09:00:00Z' }); sinon.stub(registryManager, 'shuttingDownNodes').resolves([]); @@ -159,22 +158,16 @@ describe('peerComponent tests', () => { globalState.syncthingDevicesIDCache.clear(); }); - it('frees the app when its connection is gone for every g: folder', async () => { - evidence([]); + it('frees the app when its connection is gone', async () => { + evidence({ connected: false }); expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); }); - // Canary for the test above: the same peer with one folder still connected - // is one that may still be writing it. - it('does not free the app while one g: folder is still connected to it', async () => { - evidence(['fluxweb_a']); - - expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); - }); - - it('does not free the app while the anchor\'s folder is still connected to it', async () => { - evidence(['fluxdb_a']); + // Canary for the test above. The folders are not shared with it, as for a + // peer no location lists any more: the connection alone is what keeps it. + it('does not free the app while its syncthing is still connected, whatever folders it shares', async () => { + evidence({ connected: true }); expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 3d69581ede..eae4f6135e 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -19,6 +19,7 @@ const syncthingServiceMock = { systemRestart: sinon.stub(), getConfig: sinon.stub(), getDbCompletion: sinon.stub(), + getConnectedDeviceIds: sinon.stub(), // findSyncedPeer asks for this node's own device so it can skip it: every // folder's device list begins with the local device, and its copy trivially // reports 100% complete. @@ -224,6 +225,9 @@ describe('syncthingFolderStateMachine tests', () => { syncthingServiceMock.systemRestart.resolves(); syncthingServiceMock.getConfig.reset(); syncthingServiceMock.getDbCompletion.reset(); + syncthingServiceMock.getConnectedDeviceIds.reset(); + // default: this node's syncthing is connected to no device + syncthingServiceMock.getConnectedDeviceIds.resolves(new Set()); syncthingServiceMock.getConfigDevices.reset(); // default: this node's syncthing has no device configured for the peer either, // so a cache miss stays a genuine "cannot say" rather than silently resolving @@ -1507,7 +1511,10 @@ describe('syncthingFolderStateMachine tests', () => { globalBytes: 100000, inSyncBytes: 100000, state: 'idle', receiveOnlyChangedFiles: 0, }); globalStateMock.syncthingDevicesIDCache.set('10.0.0.1:16127', 'HOLDER-DEVICE-ID'); - syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'valid', completion: 100, globalBytes: 100000 }); + syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'notSharing', completion: 100, globalBytes: 100000 }); + // Connected, though the folder is not shared with it: a holder no location + // lists is dropped from the folder while its syncthing stays connected. + syncthingServiceMock.getConnectedDeviceIds.resolves(new Set(['HOLDER-DEVICE-ID'])); const result = await stateMachine.manageFolderSyncState(mockParams); @@ -1516,13 +1523,6 @@ describe('syncthingFolderStateMachine tests', () => { // follower reaches sendreceive by its own route and that route is not what // this guard governs. expect(result.cache.designatedLeader).to.not.equal(true); - // Asked about THIS folder: the completion endpoint's aggregate form never - // sets remoteState, so a query without one reports 'unknown' and would - // veto nothing. - const asked = syncthingServiceMock.getDbCompletion.getCalls() - .find((call) => call.args[0]?.device === 'HOLDER-DEVICE-ID'); - expect(asked, 'the holder was never asked about').to.not.equal(undefined); - expect(asked.args[0].folder).to.equal('test-app'); }); it('leaves a holder that answers an error status in the election - it replied, so it is alive', async () => { From 88f9da21b86080a0df4b29268b6d2577800d1078 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 07:53:11 +0100 Subject: [PATCH 304/369] test(harness): 3310 keeps a listed primary across the standby's restart and fences it once its location expires; 3316 takes the app over beside a primary no location lists 3310: a primary whose FluxOS has stopped stays in the standby's folder while it is listed, across a restart of the standby's FluxOS that leaves it unable to ask the primary for its device, and leaves it once its location expires, while its syncthing is still connected; what it writes then reaches no holder. 3316: a primary no location lists keeps running and writing; the standby takes the app over beside it once FDM stops naming it, what each writes stays on its own side, and a holder placed after the expiry syncs the new primary's copy only. 2302 counts lookups answered from the device configured under an address. Co-Authored-By: Claude Opus 5.5 --- ...-standby-whose-calls-reach-another-node.js | 2 +- ...-went-quiet-and-then-died-is-taken-over.js | 98 +++++++++---- ...-location-lists-is-fenced-while-it-runs.js | 131 ++++++++++++++++++ 3 files changed, 204 insertions(+), 27 deletions(-) create mode 100644 test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js diff --git a/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js b/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js index ad92278b97..e73f947dea 100644 --- a/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js +++ b/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js @@ -53,7 +53,7 @@ const appDir = (name) => `/mnt/appdata/flux-apps/flux${name}_${name}`; // Where a node's syncthing monitor took each device it looked up for `name`, // counted per source. -const DEVICE_SOURCES = ['verified', 'verifiedNoDevice', 'introduced', 'withheld', 'held', 'unsigned']; +const DEVICE_SOURCES = ['verified', 'verifiedNoDevice', 'introduced', 'withheld', 'held', 'unsigned', 'configured']; async function deviceSources(client, name) { const counts = await Promise.all(DEVICE_SOURCES.map((source) => client.getDecisionCount('syncthing:deviceSource', name, source))); return Object.fromEntries(DEVICE_SOURCES.map((source, i) => [source, counts[i]])); diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js index ecd92d9d18..96d52d4fdc 100644 --- a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -1,17 +1,20 @@ // weight: heavy /* * A single-writer app's primary stops announcing while it carries on running: its - * FluxOS is gone, its syncthing and its container are not. Its location expires, - * and the standby keeps it in the folder for as long as its syncthing is connected - * and shares the folder; nothing starts beside it. + * FluxOS is gone, its syncthing and its container are not. * - * Then the machine dies. The standby drops it from the folder, and changes on the - * standby's disk to a file and a directory the primary wrote are now held by no other - * device: syncthing reverts such a change by deleting it, which a directory that is - * not empty refuses for ever. The rest of the folder is unchanged, so the standby's - * copy still reads as synced and the revert is due. The standby keeps the changes, - * takes the app over with the primary's data intact, and a new standby syncs that - * copy. + * While its location is listed it stays in the standby's folder, across a restart + * of the standby's FluxOS that leaves the standby unable to ask it for its device. + * Once its location expires the standby drops it from the folder, though its + * syncthing is still connected, and what it writes from then on reaches no holder. + * FDM still names it, so the standby stays a standby. + * + * Then the machine dies. Changes on the standby's disk to a file and a directory the + * primary wrote are held by no other device: syncthing reverts such a change by + * deleting it, which a directory that is not empty refuses for ever. The rest of the + * folder is unchanged, so the standby's copy still reads as synced and the revert is + * due. The standby keeps the changes, takes the app over with the primary's data + * intact, and a new standby syncs that copy. * * A = the standby, B = the primary, C = the new standby. */ @@ -22,7 +25,9 @@ import { expect } from 'chai'; import { placeSilentPrimaryApp, A, B, C, } from '../framework/silent-primary.js'; -import { crashFluxos, execInContainer, stopAppContainer } from '../framework/container.js'; +import { + crashFluxos, execInContainer, releaseFluxos, stopAppContainer, +} from '../framework/container.js'; import { getDeviceId, getConnectedDevices, getFolderConfig, getFileInfo, isDaemonUp, readPath, scanFolder, stopDaemon, } from '../framework/syncthing-real.js'; @@ -36,14 +41,18 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const SIGTERM_EXPIRY_S = 120; // The harness's syncthing monitor passes every 3s; this is several of them. const MONITOR_PASSES_MS = 30000; -// A machine that dies closes none of its connections, so the standby's syncthing -// holds them until its own receive timeout (5 min, not compressed by the harness). -// Then a few election passes and a container start. +// A few election passes and a container start. const TAKEOVER_MS = 10 * 60 * 1000; // syncthing's FlagLocalReceiveOnly const RECEIVE_ONLY_CHANGE_FLAG = 8; const SAVE = 'appdata/save-b'; const SAVE_CONTENT = 'written by the primary'; +const LISTED = 'appdata/written-while-listed'; +const FENCED = 'appdata/written-after-expiry'; +// Lookups of the primary's device on the standby, each one pass. +const LOOKUP_PASSES = 3; +// Where a node's syncthing monitor took each device it looked up, counted per source. +const DEVICE_SOURCES = ['verified', 'verifiedNoDevice', 'introduced', 'withheld', 'held', 'unsigned', 'configured']; function step(what) { console.log(`# 3310 ${new Date().toISOString()} ${what}`); @@ -58,6 +67,18 @@ describe('a primary that went quiet and then died is taken over', function () { const client = (i) => fleet.env.clients[i]; let primaryDevice; let afterDeath; + // How long the primary's write took to reach the standby while it was listed. + let listedSyncMs; + const lookups = async () => { + const counts = await Promise.all(DEVICE_SOURCES.map((source) => client(A) + .getDecisionCount('syncthing:deviceSource', `${client(B).ip}:16127`, source))); + return counts.reduce((sum, n) => sum + n, 0); + }; + async function writeOnPrimary(path, content) { + const wrote = await execInContainer(client(B).container, `printf '${content}' > ${root}/${path}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on the primary: ${wrote.output}`); + await scanFolder(client(B), folder); + } before(async function () { this.timeout(900000); @@ -77,21 +98,48 @@ describe('a primary that went quiet and then died is taken over', function () { await fleet?.env?.teardown(); }); - it('keeps a primary whose location expired in the folder while it is connected', async function () { - this.timeout(300000); + it('keeps a listed primary whose FluxOS does not answer in the folder, across a restart of the standby\'s FluxOS', async function () { + this.timeout(600000); step('the primary stops announcing; its syncthing and app carry on'); await crashFluxos(client(B).container, { hold: true }); expect(await isDaemonUp(client(B)), 'the primary\'s syncthing went down with its FluxOS, so nothing here is tested').to.equal(true); + + step('the standby\'s FluxOS restarts, holding nothing for the primary\'s device'); + await crashFluxos(client(A).container, { hold: true }); + await releaseFluxos(client(A).container); + await waitFor(async () => (await lookups()) >= LOOKUP_PASSES, { + timeout: 240000, interval: 3000, label: `${LOOKUP_PASSES} lookups of the primary's device on the restarted standby`, + }); + const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; + expect(devices, 'the standby dropped a listed primary from the folder once it could not ask it for its device').to.include(primaryDevice); + expect(await client(A).getDecisionCount('syncthing:deviceSource', `${client(B).ip}:16127`, 'configured'), + 'the standby never took the primary\'s device from its own configuration').to.be.at.least(1); + + const startedAt = Date.now(); + await writeOnPrimary(LISTED, 'written while listed'); + await waitFor(async () => (await readPath(client(A), `${root}/${LISTED}`)) === 'written while listed', { + timeout: 120000, interval: 1000, label: 'the listed primary\'s write reached the standby', + }); + listedSyncMs = Date.now() - startedAt; + expect(fleet.twoRan, 'the standby started beside a primary that still runs').to.deep.equal([]); + }); + + it('drops the primary from the folder once its location expires, and what it writes then reaches no holder', async function () { + this.timeout(300000); const expired = await Promise.all([A, C].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); expect(expired[0], 'the standby held no location for the primary, so none expired').to.be.at.least(1); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) + .some((d) => d.deviceID === primaryDevice), { + timeout: 120000, interval: 3000, label: 'the standby drops the primary no location lists from the folder', + }); - await sleepUnlessInfraDead(MONITOR_PASSES_MS); + step('the primary writes after its location expired'); + await writeOnPrimary(FENCED, 'written after expiry'); + // Connected and fenced: the write would arrive over this connection if the + // folder were still shared with it. expect(await getConnectedDevices(client(A)), 'the primary\'s syncthing is not connected to the standby').to.include(primaryDevice); - const host = (ip) => String(ip).split(':')[0]; - const primaryRows = (await dbClient(A + 1).getAppLocations(appName)).filter((row) => host(row.ip) === host(client(B).ip)); - expect(primaryRows.length, 'a location for the primary came back').to.equal(0); - const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; - expect(devices, 'the standby dropped a connected primary from the folder').to.include(primaryDevice); + await sleepUnlessInfraDead(Math.max(MONITOR_PASSES_MS, 5 * listedSyncMs)); + expect(await readPath(client(A), `${root}/${FENCED}`), 'the write of a primary no location lists reached the standby').to.equal(null); expect(fleet.twoRan, 'the standby started beside a primary that still runs').to.deep.equal([]); }); @@ -100,10 +148,6 @@ describe('a primary that went quiet and then died is taken over', function () { step('the primary\'s machine dies'); await stopAppContainer(client(B).container, appName, appName); await stopDaemon(client(B)); - await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) - .some((d) => d.deviceID === primaryDevice), { - timeout: 600000, interval: 5000, label: 'the standby drops the dead primary from the folder', - }); step('the standby\'s copy of the primary\'s data changes on its disk'); afterDeath = client(A).getLastEventId(); @@ -132,6 +176,7 @@ describe('a primary that went quiet and then died is taken over', function () { timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', }); expect(await readPath(client(A), `${root}/${SAVE}`), 'the primary\'s file is gone from the standby').to.equal(SAVE_CONTENT); + expect(await readPath(client(A), `${root}/${FENCED}`), 'the dead primary\'s write from after its location expired reached the standby').to.equal(null); expect((await readPath(client(A), `${root}/appdata/seed-data`))?.trim(), 'the primary\'s data is gone from the standby').to.equal('seeded'); const kept = client(A).getEventBuffer().some((e) => e.id > afterDeath && e.event === 'syncthing:localChangesKept' && e.data?.folder === folder); @@ -147,5 +192,6 @@ describe('a primary that went quiet and then died is taken over', function () { timeout: 300000, interval: 5000, label: 'the new standby holds the primary\'s data', }); await client(C).waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 240000, { afterId: afterInstall }); + expect(await readPath(client(C), `${root}/${FENCED}`), 'the dead primary\'s write from after its location expired reached the new standby').to.equal(null); }); }); diff --git a/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js b/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js new file mode 100644 index 0000000000..05e46f99e3 --- /dev/null +++ b/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js @@ -0,0 +1,131 @@ +// weight: heavy +/* + * A single-writer app's primary stops announcing while it carries on running: its + * FluxOS is gone, its syncthing and its container are not. Its location expires. + * + * The standby drops it from the folder though its syncthing is still connected, so + * what it writes from then on reaches no holder. Once FDM stops naming it, the + * standby takes the app over beside it: a primary no location lists is out of the + * app. What each writes stays on its own side, and a holder placed after the + * location expired syncs the standby's copy and nothing of the old primary's. + * + * A = the standby, B = the primary, C = the holder placed later. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { crashFluxos, execInContainer } from '../framework/container.js'; +import { + getDeviceId, getConnectedDevices, getFolderConfig, isDaemonUp, readPath, scanFolder, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { installOnNodes } from '../framework/reconciler-suite.js'; +import { clearMaster } from '../framework/fdm-control.js'; +import { waitFor } from '../framework/wait.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// The harness's syncthing monitor passes every 3s; this is several of them. +const MONITOR_PASSES_MS = 30000; +// A few election passes and a container start. +const TAKEOVER_MS = 10 * 60 * 1000; +const SAVE = 'appdata/save-b'; +const SAVE_CONTENT = 'written by the primary'; +const FENCED = 'appdata/written-after-expiry'; +const FENCED_LATER = 'appdata/written-beside-the-new-primary'; +const NEW_PRIMARY = 'appdata/written-by-the-new-primary'; + +function step(what) { + console.log(`# 3316 ${new Date().toISOString()} ${what}`); +} + +describe('a primary no location lists is fenced while it runs', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2efenced${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let primaryDevice; + // How long the primary's write took to reach the standby while it was listed. + let listedSyncMs; + + async function write(i, path, content) { + const wrote = await execInContainer(client(i).container, `printf '${content}' > ${root}/${path}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on node ${i}: ${wrote.output}`); + await scanFolder(client(i), folder); + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + primaryDevice = await getDeviceId(client(B)); + const startedAt = Date.now(); + await write(B, SAVE, SAVE_CONTENT); + await waitFor(async () => (await readPath(client(A), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 240000, interval: 1000, label: 'the primary\'s second file reached the standby', + }); + listedSyncMs = Date.now() - startedAt; + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('drops the primary from the standby\'s folder once its location expires, though it still runs', async function () { + this.timeout(300000); + step('the primary stops announcing; its syncthing and app carry on'); + await crashFluxos(client(B).container, { hold: true }); + expect(await isDaemonUp(client(B)), 'the primary\'s syncthing went down with its FluxOS, so nothing here is tested').to.equal(true); + const expired = await Promise.all([A, C].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); + expect(expired[0], 'the standby held no location for the primary, so none expired').to.be.at.least(1); + await sleepUnlessInfraDead(MONITOR_PASSES_MS); + + step('the primary writes after its location expired'); + await write(B, FENCED, 'written after expiry'); + await sleepUnlessInfraDead(Math.max(MONITOR_PASSES_MS, 5 * listedSyncMs)); + expect(await readPath(client(A), `${root}/${FENCED}`), 'the write of a primary no location lists reached the standby').to.equal(null); + const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; + expect(devices, 'the standby kept a primary no location lists in the folder').to.not.include(primaryDevice); + // Connected and fenced: the write would have arrived over this connection. + expect(await getConnectedDevices(client(A)), 'the primary\'s syncthing is not connected to the standby').to.include(primaryDevice); + expect(await fleet.isUp(B), 'the primary\'s app stopped, so nothing here is tested').to.equal(true); + }); + + it('takes the app over on the standby once FDM stops naming the primary, beside the primary still running it', async function () { + this.timeout(TAKEOVER_MS + 60000); + await clearMaster(appName); + step('waiting for the takeover'); + await waitFor(() => fleet.isUp(A), { + timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', + }); + expect(await client(A).getDecisionCount('masterSlave:decision', fleet.identifier, 'previousPrimaryUnlisted'), + 'the standby took over without reading the primary as out of the app').to.be.at.least(1); + expect(await fleet.isUp(B), 'the old primary stopped, so nothing here is tested').to.equal(true); + expect(await readPath(client(A), `${root}/${SAVE}`), 'the old primary\'s data is gone from the new primary').to.equal(SAVE_CONTENT); + }); + + it('keeps what each primary writes on its own side, and a holder placed later syncs only the new primary\'s copy', async function () { + this.timeout(900000); + await write(A, NEW_PRIMARY, 'written by the new primary'); + await write(B, FENCED_LATER, 'written beside the new primary'); + + step('a holder is placed after the old primary\'s location expired'); + await installOnNodes(fleet.env, fleet.app, [C]); + await waitFor(async () => (await readPath(client(C), `${root}/${NEW_PRIMARY}`)) === 'written by the new primary', { + timeout: 300000, interval: 5000, label: 'the new primary\'s write reached the holder placed later', + }); + expect(await readPath(client(C), `${root}/${SAVE}`), 'the holder placed later lacks the data from before the expiry').to.equal(SAVE_CONTENT); + expect(await readPath(client(C), `${root}/${FENCED}`), 'the old primary\'s write after its location expired reached the holder placed later').to.equal(null); + expect(await readPath(client(C), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the holder placed later').to.equal(null); + expect(await readPath(client(A), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the new primary').to.equal(null); + expect(await readPath(client(B), `${root}/${NEW_PRIMARY}`), 'the new primary\'s write reached the old primary').to.equal(null); + }); +}); From 25f900c255f31d752aead7fe22024794be13aa73 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 08:02:58 +0100 Subject: [PATCH 305/369] fix(syncthing): a standby does not start beside a remembered primary no location lists while its syncthing is connected A remembered primary is asked whether or not a location lists it, and its silence is judged by its syncthing connection. A primary whose location lapsed has been dropped from the app's folders, so what it writes reaches no holder, and it may still be running the app: while this node's syncthing is connected to it, this node starts nothing. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 13 +---- tests/unit/advancedWorkflows.test.js | 48 ++++--------------- 2 files changed, 11 insertions(+), 50 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index c5f24f1c26..9854e662a5 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -5404,17 +5404,8 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // primary this node cannot read keeps the component. It is the // instance most likely to still hold the volume, and electing over // it is exactly the split-brain this branch is reached to avoid. - // A previous primary no location lists is out of the app and is not - // asked: its location expired, every holder has dropped it from the - // app's folders, and what it writes reaches none of them. - let previousMasterState = PeerComponent.NOT_RUNNING; - if (previousMasterNode) { - // eslint-disable-next-line no-await-in-loop - previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); - } else { - fluxEventBus.count('masterSlave:decision', identifier, 'previousPrimaryUnlisted'); - log.info(`masterSlaveApps: app:${installedApp.name} previous primary ${previousMasterAddr} holds no location - it is out of the app`); - } + // eslint-disable-next-line no-await-in-loop + const previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); if (previousMasterState !== PeerComponent.NOT_RUNNING) { if (previousMasterState === PeerComponent.RUNNING) withdrawSeedMarks(members, 'the previous primary runs it'); if (resuming && previousMasterState === PeerComponent.RUNNING) { diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 1006e43e88..5111c05fc9 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -3913,14 +3913,11 @@ describe('advancedWorkflows tests', () => { axiosGetStub.callsFake(peerAnswers({ held: [`flux${appName}`] })); await runPass(); - // FDM names nobody; its address is silent, and this node's syncthing is - // connected to its device. + // Its location lapses and FDM names nobody; its address is silent, and this + // node's syncthing is connected to its device. serviceHelperStub.resetBehavior(); serviceHelperStub.resolves(fdmNoPrimary()); - registryManagerStub.resolves([ - { name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }, - { name: appName, ip: PRIMARY, runningSince: '2026-01-01T00:00:00.000Z' }, - ]); + registryManagerStub.resolves([{ name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }]); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('silent')); peerSyncthingSays(PRIMARY, 'connected'); @@ -3932,15 +3929,13 @@ describe('advancedWorkflows tests', () => { expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); - // Its location expired, so every holder has dropped it from the app's folders - // and what it writes reaches none of them: it is out of the app, whatever its - // syncthing still says. - it('takes a remembered primary no location lists as out of the app, without asking it', async () => { - const appName = 'unlistedprimary'; + // Its location lapsed, so this node has dropped it from the app's folders; its + // syncthing is still connected to this one, and it may still be running the app. + it('does not start beside a remembered primary no location lists while its syncthing is connected, though no folder is shared with it', async () => { + const appName = 'fencedprimary'; const PRIMARY = '192.168.1.90:16127'; sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); const logInfo = sinon.stub(log, 'info'); - const count = sinon.stub(fluxEventBus, 'count'); const runPass = electionFixture(appName, [PRIMARY]); serviceHelperStub.resetBehavior(); serviceHelperStub.resolves(fdmNames('192.168.1.90')); @@ -3954,33 +3949,8 @@ describe('advancedWorkflows tests', () => { axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('silent')); peerSyncthingSays(PRIMARY, 'connected'); - logInfo.resetHistory(); - await runPass(); - - expect(axiosGetStub.getCalls().some((call) => String(call.args[0]).includes(PRIMARY)), 'the unlisted primary was asked').to.equal(false); - sinon.assert.calledWith(count, 'masterSlave:decision', appName, 'previousPrimaryUnlisted'); - expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); - }); - - // Canary for the test above: listed, the same silent and connected primary - // keeps the app. - it('keeps the app for a remembered primary a location still lists, silent and connected', async () => { - const appName = 'listedprimary'; - const PRIMARY = '192.168.1.90:16127'; - sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); - const logInfo = sinon.stub(log, 'info'); - const runPass = electionFixture(appName, [PRIMARY]); - serviceHelperStub.resetBehavior(); - serviceHelperStub.resolves(fdmNames('192.168.1.90')); - axiosGetStub.resetBehavior(); - axiosGetStub.callsFake(peerAnswers({ held: [`flux${appName}`] })); - await runPass(); - - serviceHelperStub.resetBehavior(); - serviceHelperStub.resolves(fdmNoPrimary()); - axiosGetStub.resetBehavior(); - axiosGetStub.rejects(new Error('silent')); - peerSyncthingSays(PRIMARY, 'connected'); + // syncthing reports a connected device's last-seen as now + syncthingDeviceStatsStub.resolves({ [`DEVICE-${PRIMARY}`]: { lastSeen: new Date().toISOString() } }); logInfo.resetHistory(); await runPass(); From b8295e81782ef59823a3bea699be84d018261e1e Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 08:03:33 +0100 Subject: [PATCH 306/369] test(harness): 3316 starts nothing on the standby beside a primary no location lists while its syncthing is connected The standby drops the old primary from the folder once its location expires, so what it writes reaches no holder; while the old primary's syncthing is connected the standby starts nothing, though FDM no longer names it; and a holder placed after the expiry syncs the standby's copy and nothing the old primary wrote since. Co-Authored-By: Claude Opus 5.5 --- ...-location-lists-is-fenced-while-it-runs.js | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js b/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js index 05e46f99e3..476da79956 100644 --- a/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js +++ b/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js @@ -4,10 +4,10 @@ * FluxOS is gone, its syncthing and its container are not. Its location expires. * * The standby drops it from the folder though its syncthing is still connected, so - * what it writes from then on reaches no holder. Once FDM stops naming it, the - * standby takes the app over beside it: a primary no location lists is out of the - * app. What each writes stays on its own side, and a holder placed after the - * location expired syncs the standby's copy and nothing of the old primary's. + * what it writes from then on reaches no holder. The standby starts nothing beside + * it while that connection is up, though FDM no longer names it: it may still be + * running the app. A holder placed after the location expired syncs the standby's + * copy and nothing the old primary wrote since. * * A = the standby, B = the primary, C = the holder placed later. */ @@ -32,13 +32,13 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const SIGTERM_EXPIRY_S = 120; // The harness's syncthing monitor passes every 3s; this is several of them. const MONITOR_PASSES_MS = 30000; -// A few election passes and a container start. -const TAKEOVER_MS = 10 * 60 * 1000; +// Several election passes, each of which would start the app on the standby +// if it read the old primary as gone. +const ELECTION_PASSES_MS = 3 * 60 * 1000; const SAVE = 'appdata/save-b'; const SAVE_CONTENT = 'written by the primary'; const FENCED = 'appdata/written-after-expiry'; -const FENCED_LATER = 'appdata/written-beside-the-new-primary'; -const NEW_PRIMARY = 'appdata/written-by-the-new-primary'; +const FENCED_LATER = 'appdata/written-before-a-holder-is-placed'; function step(what) { console.log(`# 3316 ${new Date().toISOString()} ${what}`); @@ -99,33 +99,31 @@ describe('a primary no location lists is fenced while it runs', function () { expect(await fleet.isUp(B), 'the primary\'s app stopped, so nothing here is tested').to.equal(true); }); - it('takes the app over on the standby once FDM stops naming the primary, beside the primary still running it', async function () { - this.timeout(TAKEOVER_MS + 60000); + it('starts nothing on the standby while the old primary\'s syncthing is connected, though FDM no longer names it', async function () { + this.timeout(ELECTION_PASSES_MS + 120000); + const before = await fleet.verdicts(A, 'connectionAlive'); await clearMaster(appName); - step('waiting for the takeover'); - await waitFor(() => fleet.isUp(A), { - timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', - }); - expect(await client(A).getDecisionCount('masterSlave:decision', fleet.identifier, 'previousPrimaryUnlisted'), - 'the standby took over without reading the primary as out of the app').to.be.at.least(1); - expect(await fleet.isUp(B), 'the old primary stopped, so nothing here is tested').to.equal(true); - expect(await readPath(client(A), `${root}/${SAVE}`), 'the old primary\'s data is gone from the new primary').to.equal(SAVE_CONTENT); + step('watching the standby\'s election'); + await sleepUnlessInfraDead(ELECTION_PASSES_MS); + expect(await fleet.verdicts(A, 'connectionAlive'), + 'the standby never judged the old primary\'s silence, so nothing here is tested').to.be.above(before); + expect(await fleet.isUp(A), 'the standby started the app beside the old primary still running it').to.equal(false); + expect(await fleet.isUp(B), 'the old primary\'s app stopped, so nothing here is tested').to.equal(true); }); - it('keeps what each primary writes on its own side, and a holder placed later syncs only the new primary\'s copy', async function () { + it('gives a holder placed after the expiry the standby\'s copy and nothing the old primary wrote since', async function () { this.timeout(900000); - await write(A, NEW_PRIMARY, 'written by the new primary'); - await write(B, FENCED_LATER, 'written beside the new primary'); + await write(B, FENCED_LATER, 'written before a holder is placed'); step('a holder is placed after the old primary\'s location expired'); await installOnNodes(fleet.env, fleet.app, [C]); - await waitFor(async () => (await readPath(client(C), `${root}/${NEW_PRIMARY}`)) === 'written by the new primary', { - timeout: 300000, interval: 5000, label: 'the new primary\'s write reached the holder placed later', + await waitFor(async () => (await readPath(client(C), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 300000, interval: 5000, label: 'the standby\'s copy reached the holder placed later', }); - expect(await readPath(client(C), `${root}/${SAVE}`), 'the holder placed later lacks the data from before the expiry').to.equal(SAVE_CONTENT); + await sleepUnlessInfraDead(Math.max(MONITOR_PASSES_MS, 5 * listedSyncMs)); expect(await readPath(client(C), `${root}/${FENCED}`), 'the old primary\'s write after its location expired reached the holder placed later').to.equal(null); expect(await readPath(client(C), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the holder placed later').to.equal(null); - expect(await readPath(client(A), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the new primary').to.equal(null); - expect(await readPath(client(B), `${root}/${NEW_PRIMARY}`), 'the new primary\'s write reached the old primary').to.equal(null); + expect(await readPath(client(A), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the standby').to.equal(null); + expect(await fleet.isUp(A), 'the standby started the app beside the old primary still running it').to.equal(false); }); }); From 7aa0a27bd27d937c88a30e443b78f9ef6c21835d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 08:35:38 +0100 Subject: [PATCH 307/369] revert(syncthing): a silent peer's liveness is read per folder, as before Reverts 0a747bbd7e and 8a4749c6fe. syncthing reports a connected device's remoteState as valid for every folder that device offers, whether or not this node still shares the folder with it, so a holder dropped from a folder whose syncthing still offers it reads as connected, and the standby starts nothing beside it. Co-Authored-By: Claude Opus 5.5 --- .../services/appMonitoring/peerComponent.js | 10 ++- .../appMonitoring/peerFolderLiveness.js | 29 ++++--- .../syncthingFolderStateMachine.js | 4 +- ZelBack/src/services/syncthingService.js | 14 +-- tests/unit/advancedWorkflows.test.js | 87 +++++++------------ tests/unit/peerComponent.test.js | 29 ++++--- .../unit/syncthingFolderStateMachine.test.js | 16 ++-- 7 files changed, 83 insertions(+), 106 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/peerComponent.js b/ZelBack/src/services/appMonitoring/peerComponent.js index e8505d72f9..cd9d571bb9 100644 --- a/ZelBack/src/services/appMonitoring/peerComponent.js +++ b/ZelBack/src/services/appMonitoring/peerComponent.js @@ -192,19 +192,21 @@ async function peerComponentState(peerSocketAddr, { /** * What a peer that did not reply at all can be shown to be doing with an app's * g: components: free only when this node's own syncthing shows the peer's - * connection gone. A peer still connected may still be writing them. + * connection gone for every one of their folders. A folder still connected is a + * component the peer may still be writing. * @param {string} peerSocketAddr The peer's socket address. * @param {object} ctx As peerComponentState. * @returns {Promise} NOT_RUNNING or UNKNOWN. */ async function silentPeerState(peerSocketAddr, { - identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }) { const ipToCheck = extractIp(peerSocketAddr); - const verdict = await silenceVerdict(peerSocketAddr, liveness); + const verdicts = await Promise.all(appIds.map((appId) => silenceVerdict(appId, peerSocketAddr, liveness))); + const verdict = verdicts.find((each) => each !== SilenceVerdict.GONE) ?? SilenceVerdict.GONE; fluxEventBus.count('peer:silenceVerdict', identifier, verdict); if (verdict === SilenceVerdict.GONE) { - log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection gone - the app is free there`); + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appIds.join(', ')} gone - the app is free there`); return PeerComponent.NOT_RUNNING; } log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent for app:${appName} and ${SILENCE_REASONS[verdict]}, will not start`); diff --git a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js index bb2490fd16..48fb81a8fa 100644 --- a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js +++ b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js @@ -349,15 +349,15 @@ async function peerDeviceId(peerIp) { } /** - * This node's syncthing's view of its own connection to a peer: - * PeerConnection.CONNECTED, DISCONNECTED, or UNKNOWN. + * This node's syncthing's view of its own connection to a peer, for one + * folder: PeerConnection.CONNECTED, DISCONNECTED, or UNKNOWN. * * FluxOS and syncthing are separate processes on that peer and fail * independently, so silence from its API is not evidence that it has stopped * writing - a FluxOS restart takes the API away for tens of seconds while - * syncthing and the container carry on. A live sync connection IS evidence, - * whatever folders it shares: a holder no location lists any more is dropped - * from the app's folders while its syncthing stays connected to this one. + * syncthing and the container carry on. A live sync connection IS evidence: + * syncthing reports remoteState 'valid' only for a device whose connection is + * open, and clears it the moment the connection closes. * * The three answers are kept apart because a silence may be acted on only with * evidence. 'disconnected' is an answer - this node's syncthing was asked @@ -380,16 +380,22 @@ async function peerDeviceId(peerIp) { * does not need to be. This is local state, maintained by the connection * itself. * + * @param {string} folderId Folder id, always passed explicitly: the completion + * endpoint's aggregate form never sets remoteState and reports 'unknown'. * @param {string} peerIp * @returns {Promise} One of PeerConnection */ -async function peerSyncthingConnection(peerIp) { +async function peerSyncthingConnection(folderId, peerIp) { const deviceId = await peerDeviceId(peerIp); if (!deviceId) return PeerConnection.UNKNOWN; - const connected = await syncthingService.getConnectedDeviceIds().catch(() => null); - if (!connected) return PeerConnection.UNKNOWN; - if (connected.has(deviceId)) return PeerConnection.CONNECTED; + const completion = await syncthingService.getDbCompletion({ + folder: folderId, + device: deviceId, + }).catch(() => null); + + if (!completion) return PeerConnection.UNKNOWN; + if (completion.remoteState === 'valid') return PeerConnection.CONNECTED; // A device this node's syncthing has paused is disconnected by this node, not // by the peer; a config that cannot be read cannot rule that out. @@ -440,12 +446,13 @@ async function peerSyncthingConnection(peerIp) { * time keeps what it holds. Once it has passed, the closed connection is * evidence again. * + * @param {string} folderId * @param {string} peerIp * @param {Object} liveness This pass's peer view * @returns {Promise} One of SilenceVerdict */ -async function silenceVerdict(peerIp, liveness) { - const connection = await peerSyncthingConnection(peerIp); +async function silenceVerdict(folderId, peerIp, liveness) { + const connection = await peerSyncthingConnection(folderId, peerIp); if (connection === PeerConnection.CONNECTED) return SilenceVerdict.CONNECTION_ALIVE; if (connection === PeerConnection.UNKNOWN) return SilenceVerdict.NO_EVIDENCE; const shutdown = await announcedShutdown(peerIp); diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 897c4de0ee..33bf5f8e26 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -798,9 +798,9 @@ async function holderIsGone(appId, holderIp, liveness) { const answer = await liveness.read(holderIp); if (answer.reachable) return false; - const verdict = await silenceVerdict(holderIp, liveness); + const verdict = await silenceVerdict(appId, holderIp, liveness); if (verdict === SilenceVerdict.CONNECTION_ALIVE) { - log.info(`holderIsGone - ${extractIp(holderIp)}'s API is silent, but this node's syncthing still holds a live connection to it; ${appId}'s holder is restarting, not gone`); + log.info(`holderIsGone - ${extractIp(holderIp)}'s API is silent, but this node's syncthing still holds a live connection to it for ${appId}; it is restarting, not gone`); fluxEventBus.publish('syncthing:holderRetained', { folder: appId, holder: holderIp, reason: 'connectionAlive' }); return false; } diff --git a/ZelBack/src/services/syncthingService.js b/ZelBack/src/services/syncthingService.js index 097843d6e0..0e81be9fba 100644 --- a/ZelBack/src/services/syncthingService.js +++ b/ZelBack/src/services/syncthingService.js @@ -943,16 +943,6 @@ async function folderCompleteOnPeers(folder, myId, connectedPeers) { return completions.every((completion) => completion.needBytes === 0 && completion.needItems === 0 && completion.needDeletes === 0); } -/** - * The devices this node's syncthing holds a live connection to, whatever folders - * they share. - * @returns {Promise>} Device ids - */ -async function getConnectedDeviceIds() { - const { connections = {} } = await request('get', '/rest/system/connections'); - return new Set(Object.keys(connections).filter((id) => connections[id]?.connected)); -} - /** * Syncthing's statistics per device, keyed by device id. lastSeen is when the * device last connected or disconnected, to the second, and "now" while it is @@ -1031,7 +1021,8 @@ async function drainFoldersToPeers(timeoutMs) { const folders = (await getConfigFolders()).filter((folder) => folder.type === 'sendreceive'); if (!folders.length) return []; const myId = await getDeviceId(); - const connectedPeers = await getConnectedDeviceIds(); + const { connections = {} } = await request('get', '/rest/system/connections'); + const connectedPeers = new Set(Object.keys(connections).filter((id) => connections[id]?.connected)); const scans = await Promise.allSettled(folders.map((folder) => scanFolder(folder.id, { timeoutMs: Math.max(1, deadline - monotonicMs()), }))); @@ -2762,7 +2753,6 @@ module.exports = { eachDbLocalChanged, dbRevert, drainFoldersToPeers, - getConnectedDeviceIds, getDeviceStats, scanFolder, // EVENTS diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 5111c05fc9..6c3abf18aa 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -803,7 +803,7 @@ describe('advancedWorkflows tests', () => { let shuttingDownNodesStub; let dockerServiceStub; let syncthingServiceStub; - let syncthingConnectedStub; + let syncthingCompletionStub; let syncthingDeviceStatsStub; let syncthingStatusStub; let syncthingDevicesStub; @@ -895,8 +895,7 @@ describe('advancedWorkflows tests', () => { // (no device ever resolved) on a healthy fleet, so a test that wants a // silence acted on has to supply the evidence for it. const syncthingServiceModule = require('../../ZelBack/src/services/syncthingService'); - sinon.stub(syncthingServiceModule, 'getDbCompletion').resolves(null); - syncthingConnectedStub = sinon.stub(syncthingServiceModule, 'getConnectedDeviceIds').resolves(new Set()); + syncthingCompletionStub = sinon.stub(syncthingServiceModule, 'getDbCompletion').resolves(null); syncthingDevicesStub = sinon.stub(syncthingServiceModule, 'getConfigDevices').resolves([]); // default: this node's syncthing has never been connected to any peer syncthingDeviceStatsStub = sinon.stub(syncthingServiceModule, 'getDeviceStats').resolves({}); @@ -1160,13 +1159,13 @@ describe('advancedWorkflows tests', () => { }; // This node's own syncthing view of a peer's device, which is what a silence is - // judged on. 'connected' is a live connection; 'closed' is a closed one if the - // connection was ever up, and no answer if it never was; and leaving the + // judged on. 'valid' is a live connection; any other state is a closed one if + // the connection was ever up, and no answer if it never was; and leaving the // device out of the cache entirely is the third answer - this node never // resolved the peer and cannot say. - const peerSyncthingSays = (peerSocketAddr, connection, { everConnected = true } = {}) => { + const peerSyncthingSays = (peerSocketAddr, remoteState, { everConnected = true } = {}) => { globalState.syncthingDevicesIDCache.set(peerSocketAddr, `DEVICE-${peerSocketAddr}`); - syncthingConnectedStub.resolves(new Set(connection === 'connected' ? [`DEVICE-${peerSocketAddr}`] : [])); + syncthingCompletionStub.resolves({ remoteState }); syncthingDeviceStatsStub.resolves(everConnected ? { [`DEVICE-${peerSocketAddr}`]: { lastSeen: '2026-09-25T14:00:00Z' } } : { [`DEVICE-${peerSocketAddr}`]: { lastSeen: '1970-01-01T00:00:00Z' } }); @@ -1625,7 +1624,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('previous primary unreachable')); - peerSyncthingSays(PEER, 'connected'); + peerSyncthingSays(PEER, 'valid'); await runPass(); sinon.assert.notCalled(releaseOperatorStart); @@ -1645,7 +1644,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('peer unreachable')); - peerSyncthingSays(PEER, 'connected'); + peerSyncthingSays(PEER, 'valid'); await runPass(); @@ -2538,7 +2537,7 @@ describe('advancedWorkflows tests', () => { peerIdentityService.askSigned.withArgs('192.168.1.90:16127').resolves({ verdict: peerIdentityService.IdentityVerdict.UNREACHABLE, reason: 'connect ECONNREFUSED', }); - peerSyncthingSays('192.168.1.90:16127', 'connected'); + peerSyncthingSays('192.168.1.90:16127', 'valid'); await runPass(); @@ -2975,7 +2974,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); // the peer answers nothing at all (the axios default), but its sync connection // to this node is open - peerSyncthingSays('192.168.1.90:16127', 'connected'); + peerSyncthingSays('192.168.1.90:16127', 'valid'); await runPass(); @@ -3010,7 +3009,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); await runPass(); @@ -3026,7 +3025,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); shuttingDownNodesStub.resolves(['192.168.1.90:16127']); await runPass(); @@ -3041,7 +3040,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); shuttingDownNodesStub.resolves(['192.168.1.90:16137']); await runPass(); @@ -3055,7 +3054,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); shuttingDownNodesStub.rejects(new Error('database unavailable')); await runPass(); @@ -3079,6 +3078,7 @@ describe('advancedWorkflows tests', () => { // nothing cached, but this node's syncthing still has the device configured // under the name the monitor gave it, and reports the connection closed syncthingDevicesStub.resolves([{ name: '192.168.1.90:16127', deviceID: 'DEVICE-DEAD-PEER' }]); + syncthingCompletionStub.resolves({ remoteState: 'unknown' }); syncthingDeviceStatsStub.resolves({ 'DEVICE-DEAD-PEER': { lastSeen: '2026-09-25T14:00:00Z' } }); await runPass(); @@ -3096,7 +3096,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed', { everConnected: false }); + peerSyncthingSays('192.168.1.90:16127', 'unknown', { everConnected: false }); await runPass(); @@ -3111,7 +3111,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingDevicesStub.resolves([{ name: '192.168.1.90:16127', deviceID: 'DEVICE-192.168.1.90:16127', paused: true }]); await runPass(); @@ -3127,7 +3127,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingStatusStub.resolves({ startTime: '2026-09-25T14:00:30Z' }); await runPass(); @@ -3143,7 +3143,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingStatusStub.resolves({ startTime: '2026-09-25T14:00:00Z' }); await runPass(); @@ -3158,7 +3158,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingStatusStub.resolves({ startTime: '2026-09-25T13:59:59Z' }); await runPass(); @@ -3182,7 +3182,7 @@ describe('advancedWorkflows tests', () => { sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); const runPass = electionFixture(appName, [PEER]); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays(PEER, 'closed', options); + peerSyncthingSays(PEER, 'unknown', options); syncthingStatusStub.resolves({ startTime }); return runPass; }; @@ -3253,7 +3253,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingStatusStub.rejects(new Error('syncthing did not answer')); await runPass(); @@ -3267,7 +3267,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingDevicesStub.rejects(new Error('syncthing did not answer')); await runPass(); @@ -3282,7 +3282,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); syncthingDeviceStatsStub.rejects(new Error('syncthing busy')); await runPass(); @@ -3300,7 +3300,7 @@ describe('advancedWorkflows tests', () => { const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); const fluxCommunication = require('../../ZelBack/src/services/fluxCommunication'); fluxCommunication.peerResponsiveness.returns({ responding: 1, total: 8 }); @@ -3321,7 +3321,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(Object.assign(new Error('Request failed with status code 500'), { response: { status: 500 } })); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); await runPass(); @@ -3536,7 +3536,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(Object.assign(new Error('Request failed with status code 500'), { response: { status: 500 } })); - peerSyncthingSays('192.168.1.90:16127', 'closed'); + peerSyncthingSays('192.168.1.90:16127', 'unknown'); await runPass(); @@ -3835,7 +3835,7 @@ describe('advancedWorkflows tests', () => { serviceHelperStub.resolves(fdmNoPrimary()); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('previous primary unreachable')); - peerSyncthingSays('192.168.1.90:16127', 'connected'); + peerSyncthingSays('192.168.1.90:16127', 'valid'); await runPass(); @@ -3920,7 +3920,7 @@ describe('advancedWorkflows tests', () => { registryManagerStub.resolves([{ name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }]); axiosGetStub.resetBehavior(); axiosGetStub.rejects(new Error('silent')); - peerSyncthingSays(PRIMARY, 'connected'); + peerSyncthingSays(PRIMARY, 'valid'); logInfo.resetHistory(); await runPass(); @@ -3929,35 +3929,6 @@ describe('advancedWorkflows tests', () => { expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); - // Its location lapsed, so this node has dropped it from the app's folders; its - // syncthing is still connected to this one, and it may still be running the app. - it('does not start beside a remembered primary no location lists while its syncthing is connected, though no folder is shared with it', async () => { - const appName = 'fencedprimary'; - const PRIMARY = '192.168.1.90:16127'; - sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); - const logInfo = sinon.stub(log, 'info'); - const runPass = electionFixture(appName, [PRIMARY]); - serviceHelperStub.resetBehavior(); - serviceHelperStub.resolves(fdmNames('192.168.1.90')); - axiosGetStub.resetBehavior(); - axiosGetStub.callsFake(peerAnswers({ held: [`flux${appName}`] })); - await runPass(); - - serviceHelperStub.resetBehavior(); - serviceHelperStub.resolves(fdmNoPrimary()); - registryManagerStub.resolves([{ name: appName, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }]); - axiosGetStub.resetBehavior(); - axiosGetStub.rejects(new Error('silent')); - peerSyncthingSays(PRIMARY, 'connected'); - // syncthing reports a connected device's last-seen as now - syncthingDeviceStatsStub.resolves({ [`DEVICE-${PRIMARY}`]: { lastSeen: new Date().toISOString() } }); - logInfo.resetHistory(); - await runPass(); - - expect(linesMatching(logInfo, 'still holds a live connection to it')).to.have.lengthOf(1); - expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); - }); - it('reads FDM naming the address this node moved from as naming this node, and starts as its primary', async () => { const appName = 'movedself'; sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); diff --git a/tests/unit/peerComponent.test.js b/tests/unit/peerComponent.test.js index e7ac69146c..c2abcdd6bd 100644 --- a/tests/unit/peerComponent.test.js +++ b/tests/unit/peerComponent.test.js @@ -142,13 +142,14 @@ describe('peerComponent tests', () => { }); describe('a peer that does not answer', () => { - // This node's syncthing saw the peer after it started, and holds a live - // connection to it or not, whatever folders it shares with it. - const evidence = ({ connected }) => { + // This node's syncthing saw the peer after it started, and holds no live + // connection to it for whichever folders the test names. + const evidence = (connectedFolders) => { sinon.stub(axios, 'get').rejects(new Error('connect ECONNREFUSED')); sinon.stub(syncthingService, 'getConfigDevices').resolves([{ name: PEER, deviceID: 'PEERDEVICE' }]); - sinon.stub(syncthingService, 'getConnectedDeviceIds').resolves(new Set(connected ? ['PEERDEVICE'] : [])); - sinon.stub(syncthingService, 'getDbCompletion').resolves({ remoteState: 'notSharing' }); + sinon.stub(syncthingService, 'getDbCompletion').callsFake(async ({ folder }) => ({ + remoteState: connectedFolders.includes(folder) ? 'valid' : 'notSharing', + })); sinon.stub(syncthingService, 'getDeviceStats').resolves({ PEERDEVICE: { lastSeen: '2026-10-16T10:00:00Z' } }); sinon.stub(syncthingService, 'getSystemStatus').resolves({ startTime: '2026-10-16T09:00:00Z' }); sinon.stub(registryManager, 'shuttingDownNodes').resolves([]); @@ -158,16 +159,22 @@ describe('peerComponent tests', () => { globalState.syncthingDevicesIDCache.clear(); }); - it('frees the app when its connection is gone', async () => { - evidence({ connected: false }); + it('frees the app when its connection is gone for every g: folder', async () => { + evidence([]); expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); }); - // Canary for the test above. The folders are not shared with it, as for a - // peer no location lists any more: the connection alone is what keeps it. - it('does not free the app while its syncthing is still connected, whatever folders it shares', async () => { - evidence({ connected: true }); + // Canary for the test above: the same peer with one folder still connected + // is one that may still be writing it. + it('does not free the app while one g: folder is still connected to it', async () => { + evidence(['fluxweb_a']); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); + }); + + it('does not free the app while the anchor\'s folder is still connected to it', async () => { + evidence(['fluxdb_a']); expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index eae4f6135e..3d69581ede 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -19,7 +19,6 @@ const syncthingServiceMock = { systemRestart: sinon.stub(), getConfig: sinon.stub(), getDbCompletion: sinon.stub(), - getConnectedDeviceIds: sinon.stub(), // findSyncedPeer asks for this node's own device so it can skip it: every // folder's device list begins with the local device, and its copy trivially // reports 100% complete. @@ -225,9 +224,6 @@ describe('syncthingFolderStateMachine tests', () => { syncthingServiceMock.systemRestart.resolves(); syncthingServiceMock.getConfig.reset(); syncthingServiceMock.getDbCompletion.reset(); - syncthingServiceMock.getConnectedDeviceIds.reset(); - // default: this node's syncthing is connected to no device - syncthingServiceMock.getConnectedDeviceIds.resolves(new Set()); syncthingServiceMock.getConfigDevices.reset(); // default: this node's syncthing has no device configured for the peer either, // so a cache miss stays a genuine "cannot say" rather than silently resolving @@ -1511,10 +1507,7 @@ describe('syncthingFolderStateMachine tests', () => { globalBytes: 100000, inSyncBytes: 100000, state: 'idle', receiveOnlyChangedFiles: 0, }); globalStateMock.syncthingDevicesIDCache.set('10.0.0.1:16127', 'HOLDER-DEVICE-ID'); - syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'notSharing', completion: 100, globalBytes: 100000 }); - // Connected, though the folder is not shared with it: a holder no location - // lists is dropped from the folder while its syncthing stays connected. - syncthingServiceMock.getConnectedDeviceIds.resolves(new Set(['HOLDER-DEVICE-ID'])); + syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'valid', completion: 100, globalBytes: 100000 }); const result = await stateMachine.manageFolderSyncState(mockParams); @@ -1523,6 +1516,13 @@ describe('syncthingFolderStateMachine tests', () => { // follower reaches sendreceive by its own route and that route is not what // this guard governs. expect(result.cache.designatedLeader).to.not.equal(true); + // Asked about THIS folder: the completion endpoint's aggregate form never + // sets remoteState, so a query without one reports 'unknown' and would + // veto nothing. + const asked = syncthingServiceMock.getDbCompletion.getCalls() + .find((call) => call.args[0]?.device === 'HOLDER-DEVICE-ID'); + expect(asked, 'the holder was never asked about').to.not.equal(undefined); + expect(asked.args[0].folder).to.equal('test-app'); }); it('leaves a holder that answers an error status in the election - it replied, so it is alive', async () => { From e201acb65c1fa498cb05e3db8591a72ad1e6b617 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 08:56:18 +0100 Subject: [PATCH 308/369] style(appQuery): no blank line doubled after installedAppsListedAt Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appQuery/appQueryService.js | 1 - 1 file changed, 1 deletion(-) diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index 6f1d1a0ae8..48c3f0c77f 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -162,7 +162,6 @@ async function installedAppsListedAt(socketAddress) { return (await registryManager.appsListedAt(socketAddress)).filter((name) => names.has(name)); } - /** * To list installed apps. Returns apps from local database. * @param {object} req Request. From f19bcfc68999a568fdc9bd61d02fab64f262f6bb Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 09:23:16 +0100 Subject: [PATCH 309/369] fix(syncthing): a holder whose volume will not mount is left out of the cold-start seed, and nobody seeds over a volume image that may hold the app's data A folder whose volume will not mount is held out, or volume gone when the mount found conclusively that the volume's image is not on the node; a pass that finds a folder unmounted without asking why keeps volume gone and holds out any other. The promoted-folders answer lists them under `heldOut` and `volumeGone`, apart from `holding`, to any caller. A node about to seed leaves a candidate that lists the folder, or one of the app's other elected folders, under either out of the choice, counted as `holderCannotSeed`. While a candidate lists it under `heldOut` and no remaining candidate holds data, nobody seeds, counted as `holderVolumeHeldOut`: that image may hold the app's only copy. A peer that lists neither is read as it always has been. Co-Authored-By: Claude Opus 5.5 --- .../services/appMonitoring/folderStates.js | 41 +++++- .../appMonitoring/peerFolderLiveness.js | 23 ++-- .../syncthingFolderStateMachine.js | 30 ++++- .../appMonitoring/syncthingMonitor.js | 20 ++- .../src/services/appQuery/appQueryService.js | 16 ++- tests/unit/appQueryService.test.js | 30 ++++- tests/unit/folderStates.test.js | 41 ++++-- tests/unit/peerFolderLiveness.test.js | 24 +++- .../unit/syncthingFolderStateMachine.test.js | 124 ++++++++++++++++++ tests/unit/syncthingMonitor.test.js | 30 ++++- 10 files changed, 330 insertions(+), 49 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/folderStates.js b/ZelBack/src/services/appMonitoring/folderStates.js index f74b6a642a..3c3c579d80 100644 --- a/ZelBack/src/services/appMonitoring/folderStates.js +++ b/ZelBack/src/services/appMonitoring/folderStates.js @@ -8,7 +8,7 @@ const dockerService = require('../dockerService'); // Every such folder has exactly one entry from its install to its removal, so a // missing entry means only that the folder is not installed here. A peer about to // seed waits on a folder this node has not looked at, ranks one it has measured, -// and reads every other as no claim. +// leaves out one this node cannot seed, and reads every other as no claim. // // Written where each fact becomes known: the install and a volume recreate, the // monitor pass, a promotion, the removal. Nothing else removes an entry. @@ -21,8 +21,12 @@ const FolderState = Object.freeze({ // Looked at, with nothing to rank: its holdings could not be read, it has sent // everything it held, or it is a ready standby. NO_CLAIM: 'noClaim', - // Held out of the monitor pass because its volume is not safe to act on. + // Held out of the monitor pass because its volume is not safe to act on, with its + // volume image still on the node: what it holds is unknown. HELD_OUT: 'heldOut', + // Held out of the monitor pass because its volume image is gone: nothing of it is + // on the node. + VOLUME_GONE: 'volumeGone', }); // What a peer is sent for a folder this node has looked at and cannot claim @@ -90,6 +94,24 @@ function heldOut(folderId) { globalState.folderStates.set(folderId, { state: FolderState.HELD_OUT }); } +/** + * Records a folder the monitor pass held out because its volume image is gone. + * @param {string} folderId + */ +function volumeGone(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.VOLUME_GONE }); +} + +/** + * Records a folder the monitor pass found not mounted without asking why. One whose + * volume image is known to be gone stays so; any other is held out. + * @param {string} folderId + */ +function notMounted(folderId) { + if (globalState.folderStates.get(folderId)?.state === FolderState.VOLUME_GONE) return; + heldOut(folderId); +} + /** * Records a folder that has turned writable: everything this node held of it is * published, so it has nothing left to claim. @@ -124,19 +146,26 @@ function claimOf(folderId) { /** * The receive-only part of the promoted-folders answer. A writable folder is in - * neither: a peer reads it from the folders this node sends. + * none of its lists: a peer reads it from the folders this node sends. Every other + * folder is in exactly one. * @param {Set} writable The folders this node holds writable. - * @returns {{holding: object, notLookedAt: string[]}} + * @returns {{holding: object, notLookedAt: string[], heldOut: string[], volumeGone: string[]}} */ function answer(writable) { const holding = {}; const notLookedAt = []; + const heldOutIds = []; + const volumeGoneIds = []; globalState.folderStates.forEach((entry, folderId) => { if (writable.has(folderId)) return; if (entry.state === FolderState.NOT_LOOKED_AT) notLookedAt.push(folderId); + else if (entry.state === FolderState.HELD_OUT) heldOutIds.push(folderId); + else if (entry.state === FolderState.VOLUME_GONE) volumeGoneIds.push(folderId); else holding[folderId] = claimOf(folderId); }); - return { holding, notLookedAt }; + return { + holding, notLookedAt, heldOut: heldOutIds, volumeGone: volumeGoneIds, + }; } module.exports = { @@ -147,9 +176,11 @@ module.exports = { heldOut, installed, measured, + notMounted, receiveOnlyFolderIds, recreated, removed, sending, settled, + volumeGone, }; diff --git a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js index 48fb81a8fa..18ee860517 100644 --- a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js +++ b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js @@ -76,7 +76,7 @@ async function holdingsRequest(socketAddr, challenge) { /** * A peer's answer as the callers read it. * @param {object} answer The `data` of the reply. - * @returns {{reachable: boolean, answerable: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[]}} + * @returns {{reachable: boolean, answerable: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[], heldOut: string[], volumeGone: string[]}} */ function readAnswer(answer) { // A peer that has not completed its first monitor pass cannot tell "I hold @@ -93,8 +93,13 @@ function readAnswer(answer) { // The folders the peer holds and has not yet measured. Absent on a peer too old to // say, which reads as none, as that node's answer always has. const notLookedAt = Array.isArray(answer?.notLookedAt) ? answer.notLookedAt : []; + // The folders the peer cannot seed because their volume will not mount, with the + // volume's image on the node (heldOut) or gone (volumeGone). Absent on a peer too + // old to say, which reads as none, as that node's answer always has. + const heldOut = Array.isArray(answer?.heldOut) ? answer.heldOut : []; + const volumeGone = Array.isArray(answer?.volumeGone) ? answer.volumeGone : []; return { - reachable: true, answerable: true, ready, folders, holding, seeding, notLookedAt, + reachable: true, answerable: true, ready, folders, holding, seeding, notLookedAt, heldOut, volumeGone, }; } @@ -104,7 +109,7 @@ function readAnswer(answer) { // may be the one holding the copy. function notReady() { return { - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } @@ -135,7 +140,7 @@ function notReady() { * somewhere else, and says nothing of the peer. * * @param {string} socketAddr Peer socket address - * @returns {Promise<{reachable: boolean, answerable: boolean, misrouted?: boolean, unproven?: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[]}>} + * @returns {Promise<{reachable: boolean, answerable: boolean, misrouted?: boolean, unproven?: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[], heldOut: string[], volumeGone: string[]}>} */ async function probePeer(socketAddr) { const ip = extractIp(socketAddr); @@ -159,13 +164,13 @@ async function probePeer(socketAddr) { if (asked.verdict === IdentityVerdict.MISROUTED) { log.info(`peerFolderLiveness - a call to ${ip} was answered by ${asked.answeredAs}; nothing it holds is known`); return { - reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } if (asked.verdict === IdentityVerdict.UNREACHABLE) { log.info(`peerFolderLiveness - could not read ${ip}: ${asked.reason}`); return { - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } @@ -179,7 +184,7 @@ async function probePeer(socketAddr) { if (!asked.mayReadUnsigned) { log.info(`peerFolderLiveness - a reply from ${ip} does not prove it came from that node; nothing it holds is known`); return { - reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } @@ -199,12 +204,12 @@ async function probePeer(socketAddr) { } log.info(`peerFolderLiveness - ${ip} answered ${error.response.status} and cannot say which folders it holds`); return { - reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } log.info(`peerFolderLiveness - could not read ${ip}: ${error.message}`); return { - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } } diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 33bf5f8e26..b20256007c 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -1538,16 +1538,31 @@ async function handleReceiveOnlyTransition(params) { // nothing and the field would fall to the address order over a holder that may hold // the owner's data, so nobody seeds this pass. const notLookedAt = []; + // Candidates that cannot seed this app because a volume of theirs will not mount, + // and so are left out of the choice. One whose volume image is still on it may hold + // the owner's data, so nobody seeds over it unless another candidate holds data; one + // whose image is gone holds nothing. + const volumeHeldOut = []; + const volumeGone = []; await Promise.all(electionList.map(async (peer) => { if (socketAddressesMatch(peer.ip, localSocketAddr)) { if (ownClaim) claims[peer.ip] = ownClaim; return; } const answer = await liveness.read(peer.ip); - if (answer?.ready && [appId, ...electedSiblings].some((folder) => (answer.notLookedAt ?? []).includes(folder))) { + const listed = (field) => answer?.ready && [appId, ...electedSiblings].some((folder) => (answer[field] ?? []).includes(folder)); + if (listed('notLookedAt')) { notLookedAt.push(peer.ip); return; } + if (listed('heldOut')) { + volumeHeldOut.push(peer.ip); + return; + } + if (listed('volumeGone')) { + volumeGone.push(peer.ip); + return; + } // Only a READY peer's claim counts - before its first monitor pass a node cannot // tell "I hold nothing" from "I have not looked". Defence in depth rather than // the thing that enforces it: a peer that has not determined its folder state @@ -1578,9 +1593,20 @@ async function handleReceiveOnlyTransition(params) { if (!cache.waitingOnLook) log.info(`handleReceiveOnlyTransition - ${appId}: ${notLookedAt.join(', ')} has not yet measured what it holds; nobody seeds until it has`); } cache.waitingOnLook = notLookedAt.length > 0; + const leftOut = [...volumeHeldOut, ...volumeGone]; + const candidates = electionList.filter((peer) => !leftOut.some((ip) => socketAddressesMatch(ip, peer.ip))); + const candidateHolds = candidates.some((peer) => (claims[peer.ip]?.bytes || 0) > 0); + const waitingOnVolume = volumeHeldOut.length > 0 && !candidateHolds; + if (leftOut.length) fluxEventBus.count('syncthing:seedDecision', appId, 'holderCannotSeed'); + if (waitingOnVolume) { + fluxEventBus.count('syncthing:seedDecision', appId, 'holderVolumeHeldOut'); + if (!cache.waitingOnVolume) log.info(`handleReceiveOnlyTransition - ${appId}: ${volumeHeldOut.join(', ')} cannot mount a volume that may hold the app's data and no other holder has data; nobody seeds until it mounts`); + } + cache.waitingOnVolume = waitingOnVolume; const electedLeader = appWriter ? appWriter.here && !aPeerHasData - : !standDownOnUnknown && !notLookedAt.length && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); + : !standDownOnUnknown && !notLookedAt.length && !waitingOnVolume + && isDesignatedLeader(candidates, localSocketAddr, aPeerHasData, claims); // The floor holderIsGone asks of a silent holder, asked of this node before // its own win can count: a node whose peers have gone quiet is the one that // fell over, and a win it confirms in that state seeds the app on a diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 77549f2b52..31652fccac 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -81,13 +81,16 @@ const appsFolder = `${appsFolderPath}/`; * @param {string} appFolder - App folder path * @param {boolean} sending - Whether syncthing currently holds this folder sendreceive * @param {string[]} unsyncedSubdirs - volume-root names the spec declared with ml: - * @returns {Promise<{isSafe: boolean, reason: string}>} Result after any repair + * @returns {Promise<{isSafe: boolean, reason: string, volumeFileMissing?: boolean}>} Result + * after any repair; volumeFileMissing when the mount found conclusively that the + * volume's image is not on this node */ async function verifyAppFolderMountWithRepair(appId, appFolder, sending, unsyncedSubdirs = []) { const verify = sending ? verifySendReceiveFolderSafety : verifyFolderMountSafety; let mountSafety = await verify(appId, appFolder, unsyncedSubdirs); if (!mountSafety.isSafe && !mountSafety.isMounted) { const mountAttempt = await volumeService.ensureAppVolumeMounted(appId); + if (mountAttempt.reason === 'volume_file_missing') return { ...mountSafety, volumeFileMissing: true }; if (mountAttempt.mounted) { // This pass replaces the record when it mounts from somewhere other than // where the record puts it, so a later one cannot re-derive the fact - @@ -135,8 +138,8 @@ function appComponents(installedApp) { * @param {Array<{appId: string, appName: string}>} extraFolders - Folder entries * verified by id alone, for folders whose owning app's spec cannot be read * @returns {Promise<{unmountedApps: Array, verifiedSafeIds: string[]}>} Apps with - * unmounted folders, and the folder ids that verified safe (so a pending - * mount-verify flag on them can be resolved) + * unmounted folders, each saying whether its volume image is gone, and the folder + * ids that verified safe (so a pending mount-verify flag on them can be resolved) */ async function checkAppFolderMounts(appsInstalled, sendingFolderIds, extraFolders = []) { const unmountedApps = []; @@ -149,7 +152,9 @@ async function checkAppFolderMounts(appsInstalled, sendingFolderIds, extraFolder verifiedSafeIds.push(appId); } else { // Folder exists but mount is not safe (empty and not mounted - likely unmounted loop device) - unmountedApps.push({ appId, appName, reason: mountSafety.reason }); + unmountedApps.push({ + appId, appName, reason: mountSafety.reason, volumeFileMissing: !!mountSafety.volumeFileMissing, + }); } }; @@ -323,7 +328,7 @@ async function processContainerData(params) { if (!markerReady) { log.warn(`processContainerData - ${appId} volume not mounted; skipping syncthing configuration this cycle`); fluxEventBus.count('syncthing:folderPass', appId, 'volumeNotMounted'); - folderStates.heldOut(appId); + folderStates.notMounted(appId); return; } @@ -740,9 +745,10 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // re-promotes once the mount is healthy. // eslint-disable-next-line no-restricted-syntax const unreadableIds = new Set(unreadableFolderEntries.map(({ appId }) => appId)); - for (const { appId, reason } of unmountedApps) { + for (const { appId, reason, volumeFileMissing } of unmountedApps) { unsafeFolderIds.add(appId); - folderStates.heldOut(appId); + if (volumeFileMissing) folderStates.volumeGone(appId); + else folderStates.heldOut(appId); fluxEventBus.count('syncthing:folderPass', appId, 'mountUnsafe'); // The container is the other writer over the bad mount, and holding it // asks nothing of syncthing: a component that syncs the folder, or one diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index 48c3f0c77f..456db6c165 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -831,9 +831,15 @@ async function callerIsFluxnode(body) { * looked at (folderStates). A peer about to seed waits on them, so a holder that has * not looked is never read as holding nothing. * + * `heldOut` and `volumeGone` are the receive-only folders this node cannot seed because + * their volume will not mount: under `heldOut` with the volume's image still on the node, + * so what it holds is unknown, and under `volumeGone` with the image gone, so it holds + * nothing. A peer about to seed leaves this node out of the choice, and does not seed + * over a `heldOut` folder unless another holder has data. + * * @param {object} req Request. * @param {object} res Response. - * @returns {object} Message carrying { ready, folders, notLookedAt, holding, seeding }. + * @returns {object} Message carrying { ready, folders, notLookedAt, heldOut, volumeGone, holding, seeding }. */ async function promotedFolderHoldings(req, res) { try { @@ -864,12 +870,18 @@ async function promotedFolderHoldings(req, res) { const entitled = authorized === true || await callerIsFluxnode(body); const ready = ids !== null; - const { holding, notLookedAt } = ready ? folderStates.answer(ids) : { holding: {}, notLookedAt: [] }; + const { + holding, notLookedAt, heldOut, volumeGone, + } = ready ? folderStates.answer(ids) : { + holding: {}, notLookedAt: [], heldOut: [], volumeGone: [], + }; const seal = peerIdentityService.answerSealer(peerIdentityService.AnswerPurpose.PROMOTED_FOLDERS, body); const response = messageHelper.createDataMessage(await seal({ ready, folders: ids === null ? [] : [...ids], notLookedAt, + heldOut, + volumeGone, ...(entitled ? { holding, seeding: Object.fromEntries(globalState.seedMarks), diff --git a/tests/unit/appQueryService.test.js b/tests/unit/appQueryService.test.js index af8f07b270..55faec2d93 100644 --- a/tests/unit/appQueryService.test.js +++ b/tests/unit/appQueryService.test.js @@ -1074,7 +1074,9 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); expect(result.holding, 'holdings went to a caller that proved nothing').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxw_w'], notLookedAt: [] }); + expect(result).to.deep.equal({ + ready: true, folders: ['fluxw_w'], notLookedAt: [], heldOut: [], volumeGone: [], + }); }); }); @@ -1091,7 +1093,9 @@ describe('appQueryService tests', () => { }); expect(result.holding, 'a body dated in the future served as a credential').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxw_w'], notLookedAt: [] }); + expect(result).to.deep.equal({ + ready: true, folders: ['fluxw_w'], notLookedAt: [], heldOut: [], volumeGone: [], + }); }); // The bound is the fleet's clocks disagreeing, which is honest and must still work. @@ -1150,7 +1154,9 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body: signed({ target: '10.0.0.8:16127' }) }); expect(result.holding, 'a signature captured in flight worked on another node').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxw_w'], notLookedAt: [] }); + expect(result).to.deep.equal({ + ready: true, folders: ['fluxw_w'], notLookedAt: [], heldOut: [], volumeGone: [], + }); }); it('withholds the holdings when this node does not know its own address', async () => { @@ -1186,7 +1192,7 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); sinon.assert.calledOnceWithExactly(peerIdentityServiceStub.answerSealer, 'promoted', body); - expect(seal.firstCall.args[0]).to.have.keys('ready', 'folders', 'notLookedAt', 'holding', 'seeding'); + expect(seal.firstCall.args[0]).to.have.keys('ready', 'folders', 'notLookedAt', 'heldOut', 'volumeGone', 'holding', 'seeding'); expect(result.sealed).to.equal(true); }); @@ -1207,8 +1213,7 @@ describe('appQueryService tests', () => { expect(signedAnswer.holding).to.not.have.property('fluxb_b'); }); - it('answers no claim, not "not looked at", for a folder held out, one it could not read, and one it has sent', async () => { - globalState.folderStates.set('fluxh_h', { state: 'heldOut' }); + it('answers no claim for a folder it could not read and one it has sent', async () => { globalState.folderStates.set('fluxn_n', { state: 'noClaim' }); const result = await appQueryService.promotedFolderHoldings({ body: signed() }); @@ -1216,11 +1221,22 @@ describe('appQueryService tests', () => { expect(result.notLookedAt).to.deep.equal([]); expect(result.holding).to.deep.equal({ fluxa_a: { bytes: 5821604997, newestModified: 200 }, - fluxh_h: { unreadable: true }, fluxn_n: { unreadable: true }, }); }); + it('lists a folder whose volume will not mount by whether its image is gone, to any caller, and claims nothing of it', async () => { + globalState.folderStates.set('fluxh_h', { state: 'heldOut' }); + globalState.folderStates.set('fluxg_g', { state: 'volumeGone' }); + + const open = await appQueryService.promotedFolderHoldings({ body: {} }); + const signedAnswer = await appQueryService.promotedFolderHoldings({ body: signed() }); + + expect(open.heldOut).to.deep.equal(['fluxh_h']); + expect(open.volumeGone).to.deep.equal(['fluxg_g']); + expect(signedAnswer.holding).to.not.have.any.keys('fluxh_h', 'fluxg_g'); + }); + it('leaves a writable folder out of both, whatever its state', async () => { globalState.folderStates.set('fluxw_w', { state: 'notLookedAt' }); diff --git a/tests/unit/folderStates.test.js b/tests/unit/folderStates.test.js index db0ffee0e8..e18deeac33 100644 --- a/tests/unit/folderStates.test.js +++ b/tests/unit/folderStates.test.js @@ -80,7 +80,9 @@ describe('folderStates tests', () => { folderStates.sending('fluxa'); - expect(folderStates.answer(new Set())).to.deep.equal({ holding: { fluxa: { unreadable: true } }, notLookedAt: [] }); + expect(folderStates.answer(new Set())).to.deep.equal({ + holding: { fluxa: { unreadable: true } }, notLookedAt: [], heldOut: [], volumeGone: [], + }); }); it('makes a recreated volume not looked at', () => { @@ -101,20 +103,43 @@ describe('folderStates tests', () => { }); describe('answer', () => { - it('lists folders not looked at apart from what it holds, and leaves out writable ones', () => { + it('lists each folder in one list by its state, and leaves out writable ones', () => { folderStateMap.set('fluxwait', { state: 'notLookedAt' }); folderStateMap.set('fluxheld', { state: 'measured', bytes: 5, newestModified: 1 }); + folderStateMap.set('fluxnone', { state: 'noClaim' }); folderStateMap.set('fluxout', { state: 'heldOut' }); + folderStateMap.set('fluxgone', { state: 'volumeGone' }); folderStateMap.set('fluxsend', { state: 'measured', bytes: 9, newestModified: 9 }); folderStateMap.set('fluxsendwait', { state: 'notLookedAt' }); - const { holding, notLookedAt } = folderStates.answer(new Set(['fluxsend', 'fluxsendwait'])); - - expect(notLookedAt).to.deep.equal(['fluxwait']); - expect(holding).to.deep.equal({ - fluxheld: { bytes: 5, newestModified: 1 }, - fluxout: { unreadable: true }, + expect(folderStates.answer(new Set(['fluxsend', 'fluxsendwait']))).to.deep.equal({ + holding: { + fluxheld: { bytes: 5, newestModified: 1 }, + fluxnone: { unreadable: true }, + }, + notLookedAt: ['fluxwait'], + heldOut: ['fluxout'], + volumeGone: ['fluxgone'], }); }); + + it('keeps a folder whose volume image is gone when it is found unmounted without asking why, and holds out any other', () => { + folderStates.volumeGone('fluxgone'); + folderStates.measured('fluxmeasured', { bytes: 5, newestModified: 1 }); + + folderStates.notMounted('fluxgone'); + folderStates.notMounted('fluxmeasured'); + + expect(folderStateMap.get('fluxgone')).to.deep.equal({ state: 'volumeGone' }); + expect(folderStateMap.get('fluxmeasured')).to.deep.equal({ state: 'heldOut' }); + }); + + it('holds out a folder whose volume image was gone once the mount check finds it on the node', () => { + folderStates.volumeGone('fluxa'); + + folderStates.heldOut('fluxa'); + + expect(folderStateMap.get('fluxa')).to.deep.equal({ state: 'heldOut' }); + }); }); }); diff --git a/tests/unit/peerFolderLiveness.test.js b/tests/unit/peerFolderLiveness.test.js index 07a36a5925..1e05c5444e 100644 --- a/tests/unit/peerFolderLiveness.test.js +++ b/tests/unit/peerFolderLiveness.test.js @@ -50,7 +50,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); expect(probes(), 'what another node holds is not this peer\'s answer').to.equal(0); }); @@ -94,7 +94,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); sinon.assert.notCalled(axiosMock.get); }); @@ -108,7 +108,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); @@ -188,7 +188,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); sinon.assert.notCalled(axiosMock.get); }); @@ -334,7 +334,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); @@ -350,7 +350,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); @@ -387,6 +387,16 @@ describe('peerFolderLiveness', () => { expect(answer.ready).to.be.false; }); + it('reads the folders a peer cannot seed because their volume will not mount, and none from a peer that does not say', async () => { + answers({ data: { data: { ready: true, folders: [], heldOut: ['fluxa_a'], volumeGone: ['fluxb_b'] } } }); + const saying = await createPeerFolderLiveness().read('10.0.0.2:16127'); + answers({ data: { data: { ready: true, folders: [] } } }); + const silent = await createPeerFolderLiveness().read('10.0.0.2:16127'); + + expect([saying.heldOut, saying.volumeGone]).to.deep.equal([['fluxa_a'], ['fluxb_b']]); + expect([silent.heldOut, silent.volumeGone]).to.deep.equal([[], []]); + }); + it('treats a malformed body as reachable but not ready', async () => { answers({ data: {} }); const liveness = createPeerFolderLiveness(); @@ -394,7 +404,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 3d69581ede..958571b87a 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2750,6 +2750,130 @@ describe('syncthingFolderStateMachine tests', () => { }); }); + // A holder whose volume will not mount cannot seed, and the address order would + // give it the seed whenever it sorts first. Here it does: 10.0.0.1, beside this + // node at 10.0.0.2. + describe('a holder that cannot mount a volume', () => { + const BROKEN = '10.0.0.1:16127'; + const coldStartBesideBroken = (brokenAnswer, { holdsData = false, others = {} } = {}) => { + mockParams.localSocketAddr = '10.0.0.2:16127'; + mockParams.appLocation.resolves([ + { ip: BROKEN, runningSince: 2000, broadcastedAt: 1000 }, + { ip: '10.0.0.2:16127', runningSince: 2000, broadcastedAt: 1000 }, + ...Object.keys(others).map((ip) => ({ ip, runningSince: 2000, broadcastedAt: 1000 })), + ]); + syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ + files: holdsData ? [localEntry('world.dat', 5821604997, '2026-09-16T10:00:00Z')] : [], + })); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 0, inSyncBytes: 0, state: 'idle', receiveOnlyChangedFiles: holdsData ? 2 : 0, + }); + axiosMock.get.callsFake(async (url) => { + const other = Object.keys(others).find((ip) => url.includes(ip.split(':')[0])); + const answer = url.includes(BROKEN.split(':')[0]) ? brokenAnswer : (others[other] ?? {}); + return { data: { data: { ready: true, folders: [], holding: {}, ...answer } } }; + }); + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { + restarted: false, numberOfExecutions: 1, leaderStreak: 5, + }); + }; + const counting = async (run) => { + const count = sinon.spy(fluxEventBus, 'count'); + try { + return await run(count); + } finally { + count.restore(); + } + }; + + it('seeds from the holder with data, leaving out one whose volume image is on it and will not mount', async () => { + await counting(async (count) => { + coldStartBesideBroken({ heldOut: ['test-app'] }, { holdsData: true }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'the holder with data did not seed').to.equal('sendreceive'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderCannotSeed'); + }); + }); + + it('seeds nothing while a holder whose volume image is on it will not mount and no other holder has data', async () => { + await counting(async (count) => { + coldStartBesideBroken({ heldOut: ['test-app'] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'an empty copy seeded over a volume that may hold the app\'s data').to.equal('receiveonly'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderVolumeHeldOut'); + }); + }); + + it('seeds an empty copy beside a holder whose volume image is gone', async () => { + await counting(async (count) => { + coldStartBesideBroken({ volumeGone: ['test-app'] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'the address order gave the seed to a holder with no volume').to.equal('sendreceive'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderCannotSeed'); + sinon.assert.neverCalledWith(count, 'syncthing:seedDecision', 'test-app', 'holderVolumeHeldOut'); + }); + }); + + // A canary for the three above: the same holder, mounted and holding nothing, + // keeps the seed by the address order. + it('leaves the seed to the address order when that holder says neither', async () => { + coldStartBesideBroken({}); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('receiveonly'); + }); + + it('leaves out a holder that cannot mount the volume of one of the app\'s other elected folders', async () => { + coldStartBesideBroken({ volumeGone: ['sibling-app'] }); + mockParams.electedSiblings = ['sibling-app']; + appQueryServiceMock.holdsComponent.resolves(false); + globalStateMock.folderStates.set('sibling-app', { state: 'measured', bytes: 0, newestModified: 0 }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + + it('seeds from the holder with data among the rest, not from the next empty address', async () => { + // 10.0.0.1 cannot mount; 10.0.0.2 (this node) holds data; 10.0.0.3 holds nothing. + coldStartBesideBroken({ heldOut: ['test-app'] }, { + holdsData: true, + others: { '10.0.0.3:16127': { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }, + }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + + it('leaves the seed to another holder\'s larger copy among the rest', async () => { + coldStartBesideBroken({ heldOut: ['test-app'] }, { + others: { '10.0.0.3:16127': { holding: { 'test-app': { bytes: 900, newestModified: 100 } } } }, + }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'an empty copy seeded beside a holder with data').to.equal('receiveonly'); + }); + + it('seeds nothing while every other holder cannot mount', async () => { + coldStartBesideBroken({ volumeGone: ['test-app'] }, { + others: { '10.0.0.3:16127': { heldOut: ['test-app'] } }, + }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('receiveonly'); + }); + }); + // Every claim is filed under the address the election list carries, because an // object key matches exactly where addresses compare tolerantly. Keyed from // anywhere else, this node stops finding its OWN claim as soon as the two diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index e6e69c8b7a..7a376f0980 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -1485,14 +1485,18 @@ describe('syncthingMonitor tests', () => { // A volume that did not mount is never measured. Left not looked at, every peer // would wait on this node for as long as the mount stays broken, and the app would // start nowhere. - it('records a folder held out because its volume did not mount as held out, which no peer waits on', async () => { + const volumeWillNotMount = (reason) => { installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); syncthingMonitorHelpersMock.requiresSyncing.returns(true); syncthingEventsConsumerMock.mountVerifyPendingIds.returns(['fluxg']); syncthingFolderStateMachineMock.verifyFolderMountSafety.resolves({ isSafe: false, isMounted: false, reason: 'empty_unmounted_directory' }); - volumeServiceMock.ensureAppVolumeMounted.resolves({ mounted: false, reason: 'volume_file_missing' }); + volumeServiceMock.ensureAppVolumeMounted.resolves({ mounted: false, reason }); syncthingServiceMock.getConfigFolders.resolves([{ id: 'fluxg', path: '/apps/fluxg', type: 'receiveonly' }]); syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + }; + + it('records a folder whose volume image is on the node and will not mount as held out, which no peer waits on', async () => { + volumeWillNotMount('volume_image_unrecognised'); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); @@ -1501,6 +1505,28 @@ describe('syncthingMonitor tests', () => { expect(notLookedAt()).to.deep.equal([]); }); + it('records a folder whose volume image is gone as volume gone', async () => { + volumeWillNotMount('volume_file_missing'); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'volumeGone' }); + }); + + it('keeps a folder whose volume image is gone so on a pass that finds it unmounted without asking why', async () => { + folderStateMap.set('fluxg', { state: 'volumeGone' }); + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingMonitorHelpersMock.ensureStfolderExists.resolves(false); + syncthingServiceMock.getConfigFolders.resolves([]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'volumeGone' }); + }); + it('records a folder whose volume directory is not mounted as held out', async () => { installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); syncthingMonitorHelpersMock.requiresSyncing.returns(true); From dfa14832032e22a0f3e9627bb631ed8c6f17c0c8 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 09:26:47 +0100 Subject: [PATCH 310/369] test(harness): 2254 leaves a holder that cannot mount its volume out of the cold-start seed A broken holder at the lowest address, by case: its image on it and another holder with data, where the holder with data seeds and the broken one returns as a standby once its volume mounts; its image on it and no other holder with data, where nobody seeds until the volume mounts; its image gone, where the other holder seeds empty; three holders with the next one empty and the last with data; and two broken holders, one of each kind. breakFolderVolume leaves a volume's image on the node with its superblock zeroed, and restoreFolderVolume puts it back. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/volume-fixture.js | 39 +++ ...ount-its-volume-is-left-out-of-the-seed.js | 225 ++++++++++++++++++ 2 files changed, 264 insertions(+) create mode 100644 test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js diff --git a/test-infra/runner/framework/volume-fixture.js b/test-infra/runner/framework/volume-fixture.js index 57ba0a5876..b76fcc3eed 100644 --- a/test-infra/runner/framework/volume-fixture.js +++ b/test-infra/runner/framework/volume-fixture.js @@ -54,6 +54,45 @@ export async function destroyFolderVolume(container, folderId) { await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)} && rm -f ${sq(folderImage(folderId))}`, `destroy the volume of ${folderId}`); } +const folderImageHeader = (folderId) => `${folderImage(folderId)}.header`; + +/** + * Leaves a component's volume image on the node with nothing able to mount it: + * unmounted, its directory no longer immutable, and the image's first 4 KiB, which + * carry its filesystem's superblock, zeroed. The bytes are kept beside the image for + * restoreFolderVolume. Waits for the loop device to release the image first, so no + * mount still open on it writes the superblock back. + * @param {string} folderId The component's identifier, flux_. + */ +export async function breakFolderVolume(container, folderId) { + const root = folderRoot(folderId); + const image = folderImage(folderId); + await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)}`, `unmount the volume of ${folderId}`); + await run( + container, + `for i in $(seq 1 60); do [ -z "$(losetup -j ${sq(image)})" ] && exit 0; sleep 1; done; losetup -j ${sq(image)}; exit 1`, + `wait for the loop device to release the image of ${folderId}`, + ); + await run( + container, + `dd if=${sq(image)} of=${sq(folderImageHeader(folderId))} bs=4096 count=1 status=none && dd if=/dev/zero of=${sq(image)} bs=4096 count=1 conv=notrunc status=none`, + `zero the superblock of ${folderId}`, + ); + const probe = await execInContainer(container, `blkid -p ${sq(image)}`); + if (probe.exitCode === 0) throw new Error(`breakFolderVolume: ${image} still reads as a filesystem: ${probe.output}`); +} + +/** + * Puts back what breakFolderVolume zeroed, so the image mounts again. + * @param {string} folderId The component's identifier, flux_. + */ +export async function restoreFolderVolume(container, folderId) { + const image = folderImage(folderId); + const header = folderImageHeader(folderId); + await run(container, `dd if=${sq(header)} of=${sq(image)} bs=4096 count=1 conv=notrunc status=none && rm -f ${sq(header)}`, `restore the superblock of ${folderId}`); + await run(container, `blkid -p ${sq(image)}`, `read the restored image of ${folderId} as a filesystem`); +} + /** * destroyFolderVolume for an app whose one component shares its name. */ diff --git a/test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js b/test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js new file mode 100644 index 0000000000..a93377f29e --- /dev/null +++ b/test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js @@ -0,0 +1,225 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart, pinHolding } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { breakFolderVolume, destroyFolderVolume, restoreFolderVolume } from '../framework/volume-fixture.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Cold starts of a g: app where a holder comes back from a restart with a volume it +// cannot mount, and its address sorts first, so the address order would give it the +// seed. It cannot seed. +// +// A holder whose volume image is still on it may hold the app's only copy: it is left +// out of the choice, a holder with data seeds, and while no other holder has data +// nobody seeds until the volume mounts. A holder whose image is gone holds nothing: +// it is left out, and the rest seed as they would without it. +// +// Every other holder is held before it measures until each broken holder has held its +// folder out, so the decisions under test are made with the broken holders' answers. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const COMPONENT = 'data'; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; +const DATA_BYTES = 5821604997; +// A broken holder whose image is on it but will not mount, and one whose image is gone. +const UNMOUNTABLE = 'unmountable'; +const GONE = 'gone'; + +describe('a holder that cannot mount its volume is left out of the seed', function () { + let env; + dumpLogsOnFailure(() => env); + const client = (i) => env.clients[i]; + const byAddress = (holders) => [...holders].sort((a, b) => (subnet.nodeIp(a + 1) < subnet.nodeIp(b + 1) ? -1 : 1)); + + // One app per case, on its own holders, decided from a cold start. + function coldStartCase(tag, { holders, broken, dataOn }) { + const appName = `e2ecannotseed${tag}${Date.now()}`; + const watch = watchGComponents(() => env, { appName, components: [COMPONENT], holders }); + const folder = watch.folderOf(COMPONENT); + const count = (i, counter, decision) => client(i).getDecisionCount(counter, folder, decision); + const brokenHolders = Object.keys(broken).map(Number); + const deciders = holders.filter((i) => !brokenHolders.includes(i)); + const seeded = (i) => count(i, 'syncthing:seedDecision', 'seeded'); + + return { + folder, + oneHolderAtMost: watch.oneHolderAtMost, + seeded, + cannotSeed: (i) => count(i, 'syncthing:seedDecision', 'holderCannotSeed'), + waitedOnVolume: (i) => count(i, 'syncthing:seedDecision', 'holderVolumeHeldOut'), + passes: (i) => count(i, 'syncthing:folderPass', 'evaluated'), + heldOutPasses: async (i) => await count(i, 'syncthing:folderPass', 'mountUnsafe') + await count(i, 'syncthing:folderPass', 'volumeNotMounted'), + seeders: async () => (await Promise.all(holders.map(seeded))).map((n, k) => (n > 0 ? holders[k] : null)).filter((i) => i !== null), + + async start() { + expect(byAddress(holders)[0], 'fixture: a broken holder is not the lowest address').to.be.oneOf(brokenHolders); + await pushTestApp(appName, 'v1', 'cannotseed', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: holders.length, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart(holders, folder); + if (dataOn !== undefined) await pinHolding(dataOn, folder, DATA_BYTES); + const from = Object.fromEntries(deciders.map((i) => [i, client(i).getLastEventId()])); + await Promise.all(deciders.map((i) => client(i).holdCheckpoint(BEFORE_HOLDINGS, folder))); + await installOnNodes(env, app, holders); + await Promise.all(deciders.map((i) => client(i).waitForEvent( + 'checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, + 300000, + { afterId: from[i] }, + ))); + + // eslint-disable-next-line no-restricted-syntax + for (const i of brokenHolders) { + // eslint-disable-next-line no-await-in-loop + await crashFluxos(client(i).container, { hold: true }); + // eslint-disable-next-line no-await-in-loop + if (broken[i] === GONE) await destroyFolderVolume(client(i).container, folder); + // eslint-disable-next-line no-await-in-loop + else await breakFolderVolume(client(i).container, folder); + // eslint-disable-next-line no-await-in-loop + await releaseFluxos(client(i).container); + // eslint-disable-next-line no-await-in-loop + await waitFor(async () => await count(i, 'syncthing:folderPass', 'mountUnsafe') >= 1, { + timeout: 300000, interval: 2000, label: `node ${i} holds the folder out`, + }); + } + await Promise.all(deciders.map((i) => client(i).releaseCheckpoint(BEFORE_HOLDINGS, folder))); + }, + + // Until `done`, or SETTLE_PASSES folder passes go by on `watched`; never more than + // one holder running the app. + async settle(watched, done, label) { + const from = await this.passes(watched); + await waitHolding(async () => { + await watch.oneHolderAtMost(); + return await done() || await this.passes(watched) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label }); + }, + }; + } + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + }); + + after(async function () { + this.timeout(60000); + await Promise.all((env?.clients ?? []).map((c) => c?.releaseAllCheckpoints().catch(() => {}))); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + describe('the lowest address cannot mount a volume whose image is on it, and another holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('data', { holders: [0, 1], broken: { 0: UNMOUNTABLE }, dataOn: 1 }))); + + it('seeds on the holder with data', async function () { + this.timeout(900000); + await c.start(); + await c.settle(1, async () => await c.seeded(1) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeded(1), `the holder with data did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.cannotSeed(1), 'the holder with data never left the broken holder out').to.be.at.least(1); + expect(await c.seeded(0), 'the broken holder seeded').to.equal(0); + expect(await c.heldOutPasses(0), 'fixture: the broken holder no longer holds the folder out').to.be.at.least(1); + }); + + it('takes the broken holder back as a standby once its volume mounts', async function () { + this.timeout(600000); + const from = await c.passes(0); + await restoreFolderVolume(client(0).container, c.folder); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return await c.passes(0) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label: `the restored holder passes its folder ${SETTLE_PASSES} times` }); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([1]); + }); + }); + + describe('the lowest address cannot mount a volume whose image is on it, and no other holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('wait', { holders: [2, 3], broken: { 2: UNMOUNTABLE } }))); + + it('seeds nothing while the volume will not mount', async function () { + this.timeout(900000); + await c.start(); + await c.settle(3, async () => await c.seeded(3) >= 1, 'the other holder seeds, or the passes go by without it'); + + expect(await c.seeded(3), 'an empty copy seeded over a volume that may hold the app\'s data').to.equal(0); + expect(await c.waitedOnVolume(3), 'the other holder never waited on the broken holder\'s volume').to.be.at.least(1); + expect(await c.seeded(2), 'the broken holder seeded').to.equal(0); + }); + + it('seeds once that volume mounts', async function () { + this.timeout(600000); + await restoreFolderVolume(client(2).container, c.folder); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return (await c.seeders()).length >= 1; + }, { timeout: 300000, interval: 2000, label: 'a holder seeds once the volume mounts' }); + + expect(await c.seeders(), 'the holders that seeded').to.have.lengthOf(1); + }); + }); + + describe('the lowest address\'s volume image is gone, and no other holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('gone', { holders: [4, 5], broken: { 4: GONE } }))); + + it('seeds on the other holder, empty', async function () { + this.timeout(900000); + await c.start(); + await c.settle(5, async () => await c.seeded(5) >= 1, 'the other holder seeds, or the passes go by without it'); + + expect(await c.seeded(5), `the other holder did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.cannotSeed(5), 'the other holder never left the broken holder out').to.be.at.least(1); + expect(await c.waitedOnVolume(5), 'the other holder waited on a volume whose image is gone').to.equal(0); + expect(await c.seeded(4), 'the broken holder seeded').to.equal(0); + }); + }); + + describe('the lowest address cannot mount, the next holds nothing, and the last has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('three', { holders: [1, 3, 5], broken: { 1: UNMOUNTABLE }, dataOn: 5 }))); + + it('seeds on the holder with data, not the next address', async function () { + this.timeout(900000); + await c.start(); + await c.settle(5, async () => await c.seeded(5) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([5]); + }); + }); + + describe('two holders cannot mount, one with its image and one without, and the last has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('two', { holders: [0, 2, 4], broken: { 0: UNMOUNTABLE, 2: GONE }, dataOn: 4 }))); + + it('seeds on the holder with data', async function () { + this.timeout(900000); + await c.start(); + await c.settle(4, async () => await c.seeded(4) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([4]); + }); + }); +}); From 3c246c13b2498f2db9fb0a61a771cf08d40ee80b Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 09:46:20 +0100 Subject: [PATCH 311/369] test(harness): 2202's fleet keeps its locations through the old primary's takeover The old primary is down through the whole takeover and comes back as a standby, which a node does only while its locations have not expired; past that its boot removes every app. The fleet's location lifetime is 30 minutes, keeping production's order over the takeover. Co-Authored-By: Claude Opus 5.5 --- .../2202-single-writer-election-on-real-syncthing.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js b/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js index bae3749b50..04d978d7dc 100644 --- a/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js +++ b/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js @@ -38,6 +38,12 @@ const APP_UID = 1000; const API_PORT = 16127; const SYNCTHING_PORT = API_PORT + 2; +// The old primary is down through the whole takeover and comes back as a standby, +// which a node does only while its locations have not expired: past that its boot +// removes every app. The shared harness fleet's 63s is shorter than a takeover can +// take; production's 125 min is not, and this fleet keeps production's order. +const LOCATION_TTL_S = 30 * 60; + describe('single-writer election on real syncthing', function () { let env; dumpLogsOnFailure(() => env); @@ -78,7 +84,7 @@ describe('single-writer election on real syncthing', function () { syncthing: 'binary', tickerAutostart: false, configOverrides: { - fluxapps: { minOutgoing: 1, minIncoming: 1 }, + fluxapps: { minOutgoing: 1, minIncoming: 1, locationTtlS: LOCATION_TTL_S }, }, }); await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); From af4dd29d6a49a3a1a1ce032b393c19e253572e5c Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 11:49:41 +0100 Subject: [PATCH 312/369] fix(syncthing): a check of a standby's local changes stands while it pulls, and a revert is of the changes checked and no others A check's answer is stamped with syncthing's counts of this node's receive-only changes (files, directories, symlinks, deletions and bytes) and the devices the folder is shared with. What a standby pulls moves neither, so the answer is acted on while the primary keeps writing; a change on this node, or a device leaving the folder, makes the pass ask for the folder to be checked again. A check that finds no only copy keeps a fingerprint of every change it examined: name, history and deletion. The pass does not revert itself: it asks localChangeCheck for a revert, in the same one-at-a-time queue as the checks, and the revert reads the folder's changes again and goes ahead only while they are exactly the ones checked. Otherwise the answer is dropped and the next pass asks for a new check. The checkpoint `syncthing:beforeLocalChangeRevert` holds a revert before it reads them. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 152 +++++++++++--- .../syncthingFolderStateMachine.js | 25 +-- ZelBack/src/services/utils/fluxEventBus.js | 3 + tests/unit/localChangeCheck.test.js | 197 ++++++++++++++++-- .../unit/syncthingFolderStateMachine.test.js | 69 +++++- 5 files changed, 387 insertions(+), 59 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js index 7b0fe41a5a..b192285fc9 100644 --- a/ZelBack/src/services/appMonitoring/localChangeCheck.js +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -1,5 +1,7 @@ +const crypto = require('crypto'); const log = require('../../lib/log'); const syncthingService = require('../syncthingService'); +const messageHelper = require('../messageHelper'); const fluxEventBus = require('../utils/fluxEventBus'); // Whether a receive-only folder's local changes include the only copy of data another @@ -12,12 +14,17 @@ const fluxEventBus = require('../utils/fluxEventBus'); // answer it left. // // AN ANSWER HOLDS FOR THE FOLDER IT WAS ASKED ABOUT, AND NO OTHER. It is stamped with -// the folder as syncthing described it when the check was asked for: this node's -// sequence, each other device's sequence, and the devices the folder is shared with. -// A local write moves the first, another device's index the second, and a device -// leaving the folder - the moment a change held elsewhere becomes the only copy - -// changes the third. The pass acts on an answer only while the folder still carries -// its stamp. +// the folder as syncthing described it when the check was asked for: the counts of +// this node's changes to it, and the devices it is shared with. A change on this node +// moves the first, and a device leaving the folder - the moment a change held +// elsewhere becomes the only copy - changes the second. What this node pulls moves +// neither, so an answer stands while the primary keeps writing. The pass acts on an +// answer only while the folder still carries its stamp. +// +// A REVERT IS OF THE CHANGES CHECKED, AND NO OTHERS. Counts can stay the same while +// the changes do not, so the pass does not revert itself: it asks for a revert here, +// in the same queue as the checks, and the revert reads the folder's changes again +// and goes ahead only while they are exactly the ones the check examined. // syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. const RECEIVE_ONLY_CHANGE_FLAG = 8; @@ -25,20 +32,59 @@ const LOCAL_CHANGE_PAGE_SIZE = 1000; // How many items are asked about at once. const FILE_REQUESTS_AT_ONCE = 8; -// Checks asked for and not started: { folderId, entry, stamp, isCurrent }. +// The counts syncthing keeps of a folder's receive-only changes, which make its stamp. +const CHANGE_COUNTS = Object.freeze([ + 'receiveOnlyChangedFiles', 'receiveOnlyChangedDirectories', 'receiveOnlyChangedSymlinks', + 'receiveOnlyChangedDeletes', 'receiveOnlyChangedBytes', +]); + +// Jobs asked for and not started, at most one per folder: a check +// { kind: 'check', folderId, entry, stamp, isCurrent } or a revert +// { kind: 'revert', folderId, entry, answer, isCurrent, files, directories }. const waiting = []; let running = null; /** * The folder as syncthing describes it, in the form an answer is stamped with. - * @param {object} status The folder's db/status: `sequence` and `remoteSequence`. + * @param {object} status The folder's db/status: its counts of receive-only changes. * @param {Array<{deviceID: string}>} devices The devices the folder is shared with. * @returns {string} */ function stampOf(status, devices) { - const remote = Object.entries(status?.remoteSequence ?? {}).sort(([a], [b]) => a.localeCompare(b)); + const changes = CHANGE_COUNTS.map((count) => status?.[count] ?? null); const shared = devices.map((device) => device.deviceID).sort(); - return JSON.stringify({ sequence: status?.sequence ?? null, remote, shared }); + return JSON.stringify({ changes, shared }); +} + +/** + * Adds one listed change to a fingerprint of the folder's changes: its name, its + * history and whether it is a deletion. + * @param {crypto.Hash} hash + * @param {object} item An entry of db/localchanged. + */ +function addToFingerprint(hash, item) { + hash.update(`${item?.name}\0${(item?.version ?? []).join(',')}\0${item?.deleted ? 1 : 0}\n`); +} + +/** + * A fingerprint of every local change of a folder, as syncthing lists them now. + * @param {string} folderId + * @returns {Promise} null when the changes could not be read + */ +async function changesFingerprint(folderId) { + try { + const hash = crypto.createHash('sha256'); + for (let page = 1; ; page += 1) { + // eslint-disable-next-line no-await-in-loop + const batch = (await syncthingService.getDbLocalChanged(folderId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; + if (!Array.isArray(batch)) return page > 1 ? hash.digest('hex') : null; + batch.forEach((item) => addToFingerprint(hash, item)); + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return hash.digest('hex'); + } + } catch (error) { + log.warn(`localChangeCheck - ${folderId}: ${error.message}`); + return null; + } } /** @@ -71,29 +117,33 @@ async function firstHeldOnlyHere(folderId, names) { * made on this node, as a standby or earlier as the primary, and no other device's * history says otherwise. It is not asked about: whether it may be reverted turns * on whether another holder runs the app, not on what other devices hold. + * - `changes`: a fingerprint of every change examined, when there is no only copy; + * a revert goes ahead only while the folder's changes still have it. * @param {string} folderId - * @returns {Promise<{onlyCopy: string|false, madeHere: boolean}|null>} null when the - * changes could not be read + * @returns {Promise<{onlyCopy: string|false, madeHere: boolean, changes?: string}|null>} + * null when the changes could not be read */ async function examineLocalChanges(folderId) { try { const deviceId = await syncthingService.getDeviceId(); if (!deviceId) return null; const ownCounter = `${deviceId.substring(0, 7)}:`; + const hash = crypto.createHash('sha256'); let madeHere = false; for (let page = 1; ; page += 1) { // eslint-disable-next-line no-await-in-loop const batch = (await syncthingService.getDbLocalChanged(folderId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; - if (!Array.isArray(batch)) return page > 1 ? { onlyCopy: false, madeHere } : null; + if (!Array.isArray(batch)) return page > 1 ? { onlyCopy: false, madeHere, changes: hash.digest('hex') } : null; const fromElsewhere = []; batch.forEach((item) => { + addToFingerprint(hash, item); if ((item?.version ?? []).some((counter) => !counter.startsWith(ownCounter))) fromElsewhere.push(item.name); else madeHere = true; }); // eslint-disable-next-line no-await-in-loop const found = await firstHeldOnlyHere(folderId, fromElsewhere); if (found !== undefined) return { onlyCopy: found, madeHere }; - if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return { onlyCopy: false, madeHere }; + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return { onlyCopy: false, madeHere, changes: hash.digest('hex') }; } } catch (error) { log.warn(`localChangeCheck - ${folderId}: ${error.message}`); @@ -116,19 +166,56 @@ async function runCheck(check) { fluxEventBus.publish('syncthing:localChangesChecked', { folder: check.folderId, onlyCopy: Boolean(found.onlyCopy), madeHere: found.madeHere }); } -async function runWaitingChecks() { +/** + * Reverts a folder's local changes if they are exactly the ones its answer examined, + * and the entry has not been replaced. Otherwise the answer is dropped, so the pass + * asks for the folder to be checked again. + * @param {{folderId: string, entry: object, answer: object, isCurrent: Function, files: number, directories: number}} job + */ +async function runRevert(job) { + const { folderId, entry, answer } = job; + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT, folderId); + const now = await changesFingerprint(folderId); + if (now === null || !job.isCurrent() || entry.localChangeCheck !== answer) return; + if (now !== answer.changes) { + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + log.info(`localChangeCheck - ${folderId}: its local changes are not the ones checked; checking them again before any revert`); + return; + } + log.warn(`localChangeCheck - ${folderId} is synced but the receive-only folder has local changes (${job.files} files, ${job.directories} directories); reverting to the cluster's version`); + try { + // dataOrThrow: dbRevert answers in-band; without it a failed revert reads as reverted + messageHelper.dataOrThrow(await syncthingService.dbRevert(folderId)); + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + fluxEventBus.publish('syncthing:localChangesReverted', { folder: folderId, files: job.files, directories: job.directories }); + } catch (error) { + log.error(`localChangeCheck - revert of local changes for ${folderId} failed: ${error.message}`); + } +} + +async function runWaitingJobs() { while (waiting.length) { running = waiting.shift(); // eslint-disable-next-line no-await-in-loop - await runCheck(running); + await (running.kind === 'revert' ? runRevert(running) : runCheck(running)); } running = null; } +// The waiting job for `job`'s folder becomes `job`. +function enqueue(job) { + const index = waiting.findIndex((queued) => queued.folderId === job.folderId); + if (index >= 0) waiting[index] = job; + else waiting.push(job); + if (!running) runWaitingJobs(); +} + /** * Asks for a folder's local changes to be checked against the folder as `stamp` - * describes it. A check already asked for the folder is replaced; one running for the - * same folder, entry and stamp is left to finish. + * describes it. A job already asked for the folder is replaced; a check running for + * the same folder, entry and stamp is left to finish. * @param {string} folderId * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache; the answer * is left on it as `localChangeCheck`. @@ -136,12 +223,27 @@ async function runWaitingChecks() { * @param {Function} isCurrent Whether `entry` is still the folder's entry. */ function request(folderId, entry, stamp, isCurrent) { - if (running?.folderId === folderId && running.entry === entry && running.stamp === stamp) return; - const check = { folderId, entry, stamp, isCurrent }; - const index = waiting.findIndex((queued) => queued.folderId === folderId); - if (index >= 0) waiting[index] = check; - else waiting.push(check); - if (!running) runWaitingChecks(); + if (running?.kind === 'check' && running.folderId === folderId && running.entry === entry && running.stamp === stamp) return; + enqueue({ + kind: 'check', folderId, entry, stamp, isCurrent, + }); +} + +/** + * Asks for a folder's local changes to be reverted, if they are still the ones + * `answer` examined. A job already asked for the folder is replaced; a revert running + * for the same answer is left to finish. + * @param {string} folderId + * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache. + * @param {object} answer The answer the revert rests on, left on `entry`. + * @param {Function} isCurrent Whether `entry` is still the folder's entry. + * @param {{files: number, directories: number}} changed What the folder's status counts. + */ +function revert(folderId, entry, answer, isCurrent, { files, directories }) { + if (running?.kind === 'revert' && running.folderId === folderId && running.answer === answer) return; + enqueue({ + kind: 'revert', folderId, entry, answer, isCurrent, files, directories, + }); } /** @@ -157,7 +259,9 @@ function answerFor(entry, stamp) { module.exports = { answerFor, + changesFingerprint, examineLocalChanges, request, + revert, stampOf, }; diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index b20256007c..0a845f9b0c 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -5,7 +5,6 @@ const log = require('../../lib/log'); const dockerService = require('../dockerService'); const appReconciler = require('./appReconciler'); const appUninstaller = require('../appLifecycle/appUninstaller'); -const messageHelper = require('../messageHelper'); const syncthingService = require('../syncthingService'); const appQueryService = require('../appQuery/appQueryService'); const serviceHelper = require('../serviceHelper'); @@ -550,7 +549,8 @@ async function probeFolderSyncCompletion(folderId) { // so a folder of empty directories would read as holding data. const { globalFileBytes: globalBytes = 0, globalFiles = 0, inSyncFileBytes: inSyncBytes = 0, state, - receiveOnlyChangedFiles = 0, receiveOnlyChangedDirectories = 0, sequence, remoteSequence, + receiveOnlyChangedFiles = 0, receiveOnlyChangedDirectories = 0, receiveOnlyChangedSymlinks = 0, + receiveOnlyChangedDeletes = 0, receiveOnlyChangedBytes = 0, } = await syncthingService.getDbStatusInFileBytes(folderId); const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; @@ -569,9 +569,11 @@ async function probeFolderSyncCompletion(folderId) { // Directories are counted apart from files. A directory whose owner or mode // differs from the cluster's is a local change like any other. receiveOnlyChangedDirectories, - // Where this node's index and each other device's stood when the status was read. - sequence, - remoteSequence, + // The rest of what syncthing counts of this node's changes, which with the two + // above is what a check of them is stamped with: a pull moves none of them. + receiveOnlyChangedSymlinks, + receiveOnlyChangedDeletes, + receiveOnlyChangedBytes, // An EMPTY global index (globalBytes 0) means "unknown / not yet synced", // never "done": a node holding the only copy before its peers reconnect // reads globalBytes 0, and syncPercentage defaults to 100 there (vacuous). @@ -1315,7 +1317,8 @@ async function nudgeFolderDevices(folderId) { * publishes them. Which changes the folder holds comes from localChangeCheck, and is * acted on only while the folder is still the one it was checked against. Until then, * and while whether another holder runs the app cannot be said, the folder is neither - * reverted nor promoted. + * reverted nor promoted. The revert is localChangeCheck's, of the changes checked and + * no others. * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status @@ -1378,15 +1381,7 @@ async function revertLocalChangesIfSynced(appId, syncStatus, folderCache, { isCu } delete cache.waitingOnHolders; delete cache.keepingChanges; - log.warn(`revertLocalChangesIfSynced - ${appId} is synced but the receive-only folder has ${changed} locally changed item(s) (${files} files, ${directories} directories); reverting to the cluster's version`); - try { - // dataOrThrow: dbRevert answers in-band; without it this catch is - // dead code and a failed revert reads as reverted - messageHelper.dataOrThrow(await syncthingService.dbRevert(appId)); - fluxEventBus.publish('syncthing:localChangesReverted', { folder: appId, files, directories }); - } catch (error) { - log.error(`revertLocalChangesIfSynced - revert of local changes for ${appId} failed: ${error.message}`); - } + localChangeCheck.revert(appId, cache, answer, isCurrent, { files, directories }); return true; } diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index 1df515d472..eb356db371 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -74,6 +74,9 @@ const Checkpoint = Object.freeze({ // A check of a receive-only folder's local changes has its answer and has not // left it for the monitor pass. Keyed by folder id. SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER: 'syncthing:beforeLocalChangeAnswer', + // A revert of a receive-only folder's local changes has not yet read whether they + // are still the ones checked. Keyed by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT: 'syncthing:beforeLocalChangeRevert', // The syncthing monitor has reached a receive-only folder and not yet measured // what this node holds in it. Keyed by folder id. SYNCTHING_BEFORE_HOLDINGS: 'syncthing:beforeHoldings', diff --git a/tests/unit/localChangeCheck.test.js b/tests/unit/localChangeCheck.test.js index 616f8f9660..d20ed65c92 100644 --- a/tests/unit/localChangeCheck.test.js +++ b/tests/unit/localChangeCheck.test.js @@ -7,6 +7,7 @@ const syncthingServiceMock = { getDeviceId: sinon.stub(), getDbLocalChanged: sinon.stub(), getDbFile: sinon.stub(), + dbRevert: sinon.stub(), }; const localChangeCheck = proxyquire('../../ZelBack/src/services/appMonitoring/localChangeCheck', { '../syncthingService': syncthingServiceMock, @@ -34,27 +35,48 @@ describe('localChangeCheck tests', () => { syncthingServiceMock.getDbLocalChanged.resolves({ files: [] }); syncthingServiceMock.getDbFile.reset(); syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 } }); + syncthingServiceMock.dbRevert.reset(); + syncthingServiceMock.dbRevert.resolves({ status: 'success', data: {} }); }); + // An answer with no only copy, as runCheck leaves it. + const FINGERPRINT = sinon.match(/^[0-9a-f]{64}$/); + describe('stampOf', () => { - it('is the same for the same folder, whatever order syncthing lists it in', () => { - const one = localChangeCheck.stampOf({ sequence: 3, remoteSequence: { B: 2, A: 1 } }, [{ deviceID: 'Y' }, { deviceID: 'X' }]); - const two = localChangeCheck.stampOf({ sequence: 3, remoteSequence: { A: 1, B: 2 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]); + const COUNTS = { + receiveOnlyChangedFiles: 2, receiveOnlyChangedDirectories: 1, receiveOnlyChangedSymlinks: 0, receiveOnlyChangedDeletes: 0, receiveOnlyChangedBytes: 4096, + }; - expect(one).to.equal(two); + it('is the same for the same folder, whatever order syncthing lists its devices in', () => { + expect(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'Y' }, { deviceID: 'X' }])) + .to.equal(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'X' }, { deviceID: 'Y' }])); + }); + + // A pull moves both, and neither says anything about what this node changed. + it('does not change with this node\'s sequence or another device\'s', () => { + const devices = [{ deviceID: 'X' }]; + + expect(localChangeCheck.stampOf({ ...COUNTS, sequence: 3, remoteSequence: { A: 1 } }, devices)) + .to.equal(localChangeCheck.stampOf({ ...COUNTS, sequence: 90, remoteSequence: { A: 70 } }, devices)); }); [ - ['this node\'s sequence', { sequence: 4, remoteSequence: { A: 1 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]], - ['another device\'s sequence', { sequence: 3, remoteSequence: { A: 2 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]], - ['the devices the folder is shared with', { sequence: 3, remoteSequence: { A: 1 } }, [{ deviceID: 'X' }]], - ].forEach(([what, status, devices]) => { + ['the files this node changed', { receiveOnlyChangedFiles: 3 }], + ['the directories this node changed', { receiveOnlyChangedDirectories: 2 }], + ['the symlinks this node changed', { receiveOnlyChangedSymlinks: 1 }], + ['the deletions this node made', { receiveOnlyChangedDeletes: 1 }], + ['the bytes this node changed', { receiveOnlyChangedBytes: 8192 }], + ].forEach(([what, moved]) => { it(`changes with ${what}`, () => { - const before = localChangeCheck.stampOf({ sequence: 3, remoteSequence: { A: 1 } }, [{ deviceID: 'X' }, { deviceID: 'Y' }]); + const devices = [{ deviceID: 'X' }]; - expect(localChangeCheck.stampOf(status, devices)).to.not.equal(before); + expect(localChangeCheck.stampOf({ ...COUNTS, ...moved }, devices)).to.not.equal(localChangeCheck.stampOf(COUNTS, devices)); }); }); + + it('changes with the devices the folder is shared with', () => { + expect(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'X' }])).to.not.equal(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'X' }, { deviceID: 'Y' }])); + }); }); describe('request', () => { @@ -71,8 +93,12 @@ describe('localChangeCheck tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.getDbLocalChanged, 'fluxb_b'); first.release(); await settled(); - expect(a.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false, madeHere: false }); - expect(b.localChangeCheck).to.deep.equal({ stamp: 's1', onlyCopy: false, madeHere: false }); + sinon.assert.match(a.localChangeCheck, { + stamp: 's1', onlyCopy: false, madeHere: false, changes: FINGERPRINT, + }); + sinon.assert.match(b.localChangeCheck, { + stamp: 's1', onlyCopy: false, madeHere: false, changes: FINGERPRINT, + }); }); it('replaces a check still waiting with the newer one for the same folder', async () => { @@ -130,7 +156,7 @@ describe('localChangeCheck tests', () => { it('says a change was made here, asking nothing about it', async () => { syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); - expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: false, madeHere: true }); + sinon.assert.match(await localChangeCheck.examineLocalChanges('fluxa_a'), { onlyCopy: false, madeHere: true, changes: FINGERPRINT }); sinon.assert.notCalled(syncthingServiceMock.getDbFile); }); @@ -139,13 +165,13 @@ describe('localChangeCheck tests', () => { syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a', 1).resolves({ files: held }); syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a', 2).resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); - expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: false, madeHere: true }); + sinon.assert.match(await localChangeCheck.examineLocalChanges('fluxa_a'), { onlyCopy: false, madeHere: true, changes: FINGERPRINT }); }); it('says nothing was made here when every change names another device', async () => { syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); - expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: false, madeHere: false }); + sinon.assert.match(await localChangeCheck.examineLocalChanges('fluxa_a'), { onlyCopy: false, madeHere: false, changes: FINGERPRINT }); }); it('cannot say when an item cannot be read', async () => { @@ -155,4 +181,145 @@ describe('localChangeCheck tests', () => { expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.equal(null); }); }); + + describe('changesFingerprint', () => { + it('is the fingerprint the check left for the same changes', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + + expect(await localChangeCheck.changesFingerprint('fluxa_a')).to.equal((await localChangeCheck.examineLocalChanges('fluxa_a')).changes); + }); + + [ + ['another name', { name: 'g', version: ['PRIMARY:1', 'LOCAL-D:2'] }], + ['another history', { name: 'f', version: ['PRIMARY:1', 'LOCAL-D:3'] }], + ['a deletion', { + name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'], deleted: true, + }], + ].forEach(([what, item]) => { + it(`differs for ${what}`, async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + const before = await localChangeCheck.changesFingerprint('fluxa_a'); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [item] }); + + expect(await localChangeCheck.changesFingerprint('fluxa_a')).to.not.equal(before); + }); + }); + + it('cannot say when the changes cannot be read', async () => { + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); + + expect(await localChangeCheck.changesFingerprint('fluxa_a')).to.equal(null); + }); + }); + + describe('revert', () => { + const CHANGED = { files: 1, directories: 0 }; + // A folder no other test has stubbed reads for, page by page. + // A folder checked with one change, its answer left on its entry. + async function checked(folderId = 'fluxr_r') { + syncthingServiceMock.getDbLocalChanged.withArgs(folderId).resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + const entry = {}; + localChangeCheck.request(folderId, entry, 's1', () => true); + await settled(); + return entry; + } + + it('reverts the changes the check examined, and says so', async () => { + const entry = await checked(); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'fluxr_r'); + sinon.assert.calledWithExactly(publish, 'syncthing:localChangesReverted', { folder: 'fluxr_r', files: 1, directories: 0 }); + expect(entry).to.not.have.property('localChangeCheck'); + } finally { + publish.restore(); + } + }); + + it('does not revert changes other than the ones checked, and drops the answer', async () => { + const entry = await checked(); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').resolves({ files: [{ name: 'g', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('does not revert while the changes cannot be read, and keeps the answer', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').rejects(new Error('connect ECONNREFUSED')); + + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry.localChangeCheck).to.equal(answer); + }); + + it('does not revert a folder whose entry was replaced, or on an answer the entry no longer carries', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + + localChangeCheck.revert('fluxr_r', entry, answer, () => false, CHANGED); + await settled(); + entry.localChangeCheck = { ...answer }; + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('keeps the answer when syncthing refuses the revert, and says nothing was reverted', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + syncthingServiceMock.dbRevert.resolves({ status: 'error', data: { message: 'folder is not running' } }); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + sinon.assert.neverCalledWith(publish, 'syncthing:localChangesReverted'); + expect(entry.localChangeCheck).to.equal(answer); + } finally { + publish.restore(); + } + }); + + it('runs in the same queue as the checks, one at a time on the node', async () => { + const entry = await checked(); + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxb_b').returns(first.promise); + + localChangeCheck.request('fluxb_b', {}, 's1', () => true); + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + first.release(); + await settled(); + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + }); + + it('does not start again a revert already running on the same answer', async () => { + const entry = await checked(); + const again = held({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').returns(again.promise); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + again.release(); + await settled(); + + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + expect(syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').callCount, 'the changes were read again for a second revert').to.equal(2); + }); + }); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 958571b87a..072fd6a12f 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -959,10 +959,11 @@ describe('syncthingFolderStateMachine tests', () => { try { // The primary runs the app on another holder. peerComponentMock.componentStateOnOtherHolders.resolves('running'); - // The first pass asks for the check; the next acts on its answer. + // The first pass asks for the check; the next asks for the revert its answer allows. await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder, 'a g: folder\'s type is its primary role\'s').to.not.have.property('type'); @@ -991,10 +992,11 @@ describe('syncthingFolderStateMachine tests', () => { try { // The primary runs the app on another holder. peerComponentMock.componentStateOnOtherHolders.resolves('running'); - // The first pass asks for the check; the next acts on its answer. + // The first pass asks for the check; the next asks for the revert its answer allows. await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnce(syncthingServiceMock.dbRevert); sinon.assert.neverCalledWith(publish, 'syncthing:localChangesReverted'); @@ -2392,7 +2394,9 @@ describe('syncthingFolderStateMachine tests', () => { const checkedPass = async () => { await pass(); await checksSettled(); - return pass(); + const result = await pass(); + await checksSettled(); + return result; }; it('is neither reverted nor promoted before the check has answered', async () => { @@ -2482,9 +2486,25 @@ describe('syncthingFolderStateMachine tests', () => { expect(result.cache.restarted, 'the only copy, kept, is promoted').to.be.true; }); + // A pull moves the folder's sequences and not its counts of this node's changes, + // so the answer stands while the primary keeps writing. + it('reverts on its answer while the folder only pulls, its sequences moving every pass', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + status({ sequence: 11, remoteSequence: { 'PRIMARY-DEVICE': 6 } }); + + await pass(); + await checksSettled(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + [ - ['a local write', { sequence: 11 }], - ['another device\'s index', { remoteSequence: { 'PRIMARY-DEVICE': 6 } }], + ['another change on this node', { receiveOnlyChangedFiles: 1 }], + ['a change of what this node changed', { receiveOnlyChangedBytes: 4096 }], ].forEach(([what, moved]) => { it(`is checked again after ${what}, and not reverted on the answer from before it`, async () => { pollutedCandidate(); @@ -2495,12 +2515,50 @@ describe('syncthingFolderStateMachine tests', () => { status(moved); await pass(); + await checksSettled(); sinon.assert.notCalled(syncthingServiceMock.dbRevert); sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); }); }); + // The counts can stay the same while the changes do not: the revert reads them + // again, and goes ahead only on the ones the check examined. + it('does not revert changes other than the ones checked, and checks them before any revert', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + changes({ name: 'appdata/other', version: FROM_PRIMARY }); + onlyHere('appdata/other'); + + await pass(); + await checksSettled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(mockParams.receiveOnlySyncthingAppsCache.get('test-app'), 'the answer the revert did not rest on').to.not.have.property('localChangeCheck'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(mockParams.receiveOnlySyncthingAppsCache.get('test-app').localChangeCheck.onlyCopy).to.equal('appdata/other'); + }); + + it('does not revert when its changes cannot be read again', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); + + await pass(); + await checksSettled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + it('leaves no answer on a folder whose entry was replaced while it was checked', async () => { pollutedCandidate(); changes({ name: 'appdata/held', version: FROM_PRIMARY }); @@ -2613,6 +2671,7 @@ describe('syncthingFolderStateMachine tests', () => { othersRun('running'); await pass(); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); }); From e78fc5db4968ae47953d387f6ea12f3cce6e2668 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 11:51:24 +0100 Subject: [PATCH 313/369] test(harness): 3317 reverts a standby's changes while the primary keeps writing, and only the changes checked The primary appends every 2 seconds for the whole suite. The standby reverts its own stray and its change to a file of the primary's while the writes keep arriving; and a revert held before it reads the folder's changes again, while a further change lands, does not go ahead until the folder has been checked again, then reverts both. Co-Authored-By: Claude Opus 5.5 --- ...changes-while-the-primary-keeps-writing.js | 133 ++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js diff --git a/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js new file mode 100644 index 0000000000..6039b6e777 --- /dev/null +++ b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js @@ -0,0 +1,133 @@ +// weight: heavy +/* + * A standby's local changes are reverted while its primary keeps writing. Every pull + * moves the folder's sequences, and none moves syncthing's counts of the standby's own + * changes, so a check of them stands while the primary writes. + * + * A revert is of the changes checked and no others: one held before it reads the + * folder's changes again, while a further change lands, does not go ahead until the + * folder is checked again. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { execInContainer } from '../framework/container.js'; +import { readPath, scanFolder } from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const BEFORE_REVERT = 'syncthing:beforeLocalChangeRevert'; +// The primary writes this often for the whole suite: faster than the standby's passes. +const WRITE_EVERY_MS = 2000; +// Long enough for many passes, each finding the folder pulled since the one before. +const REVERT_MS = 5 * 60 * 1000; + +function step(what) { + console.log(`# 3317 ${new Date().toISOString()} ${what}`); +} + +describe('a standby reverts its local changes while the primary keeps writing', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2ewriting${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let writing = false; + let writer = null; + let lines = 0; + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + const linesOnStandby = async () => Number((await sh(A, `wc -l < ${root}/appdata/live.log 2>/dev/null || echo 0`)).trim()); + const eventsSince = (name, from) => client(A).getEventBuffer().filter((e) => e.id > from && e.event === name && e.data?.folder === folder); + const event = (name, from, timeout) => client(A).waitForEvent(name, (d) => d.folder === folder, timeout, { afterId: from }); + + // The primary appends a line and has syncthing scan it, every WRITE_EVERY_MS. + async function keepWriting() { + while (writing) { + lines += 1; + // eslint-disable-next-line no-await-in-loop + await sh(B, `echo line-${lines} >> ${root}/appdata/live.log`).catch(() => {}); + // eslint-disable-next-line no-await-in-loop + await scanFolder(client(B), folder).catch(() => {}); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, WRITE_EVERY_MS); }); + } + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + writing = true; + writer = keepWriting(); + await waitFor(async () => (await linesOnStandby()) >= 3, { + timeout: 240000, interval: 2000, label: 'the primary\'s writes reach the standby', + }); + }); + + // A test that fails holding a revert would hold the next test's reverts too. + afterEach(async function () { + await client(A)?.releaseAllCheckpoints().catch(() => {}); + }); + + after(async function () { + this.timeout(60000); + writing = false; + await writer; + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('reverts its changes while the primary keeps writing', async function () { + this.timeout(REVERT_MS + 120000); + const from = client(A).getLastEventId(); + step('the standby changes the mode of one of the primary\'s files and makes a stray'); + await sh(A, `chmod 600 ${root}/appdata/seed-data && echo stray > ${root}/appdata/stray-1`); + await scanFolder(client(A), folder); + const linesAtChange = await linesOnStandby(); + + await event('syncthing:localChangesReverted', from, REVERT_MS).catch(() => { + throw new Error('the standby did not revert its changes while the primary kept writing'); + }); + expect(await readPath(client(A), `${root}/appdata/stray-1`), 'the stray is still on the standby').to.equal(null); + expect((await sh(A, `stat -c %a ${root}/appdata/seed-data`)).trim(), 'the primary\'s file keeps the standby\'s mode').to.not.equal('600'); + await waitFor(async () => (await linesOnStandby()) > linesAtChange, { + timeout: 60000, interval: 2000, label: 'the primary\'s writes kept reaching the standby', + }); + }); + + it('does not revert a change made after the check until it has been checked', async function () { + this.timeout(REVERT_MS + 300000); + await client(A).holdCheckpoint(BEFORE_REVERT, folder); + const from = client(A).getLastEventId(); + await sh(A, `echo stray > ${root}/appdata/stray-2`); + await scanFolder(client(A), folder); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_REVERT && d.key === folder, REVERT_MS, { afterId: from }); + + step('a further change lands while the revert of the first is held'); + await sh(A, `echo stray > ${root}/appdata/stray-3`); + await scanFolder(client(A), folder); + await waitFor(async () => Number((await sh(A, `ls ${root}/appdata | grep -c '^stray-'`)).trim()) === 2, { + timeout: 30000, interval: 1000, label: 'both strays are on the standby', + }); + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_REVERT, folder); + + const reverted = await event('syncthing:localChangesReverted', releasedAt, REVERT_MS); + const checked = eventsSince('syncthing:localChangesChecked', releasedAt); + expect(checked.filter((e) => e.id < reverted.id), 'reverted changes it had not checked').to.not.deep.equal([]); + expect(await readPath(client(A), `${root}/appdata/stray-2`), 'the first stray is still on the standby').to.equal(null); + expect(await readPath(client(A), `${root}/appdata/stray-3`), 'the second stray is still on the standby').to.equal(null); + }); +}); From 8e249059950315edf041d5bca2d080134128ca30 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 11:57:11 +0100 Subject: [PATCH 314/369] test(harness): 3317 waits for the pull that follows a revert to put the primary's version back Co-Authored-By: Claude Opus 5.5 --- ...ts-changes-while-the-primary-keeps-writing.js | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js index 6039b6e777..301353327a 100644 --- a/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js +++ b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js @@ -100,8 +100,14 @@ describe('a standby reverts its local changes while the primary keeps writing', await event('syncthing:localChangesReverted', from, REVERT_MS).catch(() => { throw new Error('the standby did not revert its changes while the primary kept writing'); }); - expect(await readPath(client(A), `${root}/appdata/stray-1`), 'the stray is still on the standby').to.equal(null); - expect((await sh(A, `stat -c %a ${root}/appdata/seed-data`)).trim(), 'the primary\'s file keeps the standby\'s mode').to.not.equal('600'); + // A revert resets syncthing's index; the pull that follows puts the primary's + // version back. + await waitFor(async () => (await readPath(client(A), `${root}/appdata/stray-1`)) === null, { + timeout: 120000, interval: 2000, label: 'the stray is gone from the standby', + }); + await waitFor(async () => (await sh(A, `stat -c %a ${root}/appdata/seed-data`)).trim() !== '600', { + timeout: 120000, interval: 2000, label: 'the primary\'s file has the primary\'s mode again', + }); await waitFor(async () => (await linesOnStandby()) > linesAtChange, { timeout: 60000, interval: 2000, label: 'the primary\'s writes kept reaching the standby', }); @@ -127,7 +133,9 @@ describe('a standby reverts its local changes while the primary keeps writing', const reverted = await event('syncthing:localChangesReverted', releasedAt, REVERT_MS); const checked = eventsSince('syncthing:localChangesChecked', releasedAt); expect(checked.filter((e) => e.id < reverted.id), 'reverted changes it had not checked').to.not.deep.equal([]); - expect(await readPath(client(A), `${root}/appdata/stray-2`), 'the first stray is still on the standby').to.equal(null); - expect(await readPath(client(A), `${root}/appdata/stray-3`), 'the second stray is still on the standby').to.equal(null); + await waitFor(async () => (await readPath(client(A), `${root}/appdata/stray-2`)) === null + && (await readPath(client(A), `${root}/appdata/stray-3`)) === null, { + timeout: 120000, interval: 2000, label: 'both strays are gone from the standby', + }); }); }); From 73eeb7069dff1672b701ee032fa4bd4d891b8091 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 12:20:58 +0100 Subject: [PATCH 315/369] fix(syncthing): a folder whose locations cannot be read keeps what this node tells its peers of it A pass that cannot read an app's locations still holds the folder out of itself, and leaves the folder's state as it was: a measured folder keeps its measurement, and one not yet looked at stays not looked at. A read that failed says nothing about what the volume holds. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/syncthingMonitor.js | 5 +++-- tests/unit/syncthingMonitor.test.js | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 31652fccac..94eb150762 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -351,11 +351,12 @@ async function processContainerData(params) { locations = await readAppLocation(installedAppName); } catch (error) { // Without its locations the folder would be shared with no peer, and the device - // sweep would remove every peer it has: it is held out of this pass instead. + // sweep would remove every peer it has: it is held out of this pass instead. What + // this node tells its peers of the folder stays as it was: a read that failed says + // nothing about what the volume holds. log.warn(`processContainerData - ${appId} locations could not be read (${error.message}); leaving the folder as it is this cycle`); fluxEventBus.count('syncthing:folderPass', appId, 'locationsUnread'); unsafeFolderIds.add(appId); - folderStates.heldOut(appId); return; } locations = sortAndFilterLocations(locations, localSocketAddr); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index 7a376f0980..354aa637bb 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -1539,17 +1539,32 @@ describe('syncthingMonitor tests', () => { expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); }); - it('records a folder whose locations could not be read as held out', async () => { + // A read that failed says nothing about what the volume holds. + const locationsUnreadable = () => { installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); syncthingMonitorHelpersMock.requiresSyncing.returns(true); syncthingMonitorHelpersMock.ensureStfolderExists.resolves(true); dbHelperMock.findInDatabase.rejects(new Error('database unavailable')); syncthingServiceMock.getConfigFolders.resolves([]); + }; + + it('leaves what it measured of a folder whose locations could not be read', async () => { + folderStateMap.set('fluxg', { state: 'measured', bytes: 10, newestModified: 1 }); + locationsUnreadable(); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); - expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'measured', bytes: 10, newestModified: 1 }); + }); + + it('leaves a folder it has not looked at whose locations could not be read as not looked at, which peers wait on', async () => { + locationsUnreadable(); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'notLookedAt' }); }); }); From a55b2ed0633204d1e607d9876fde69c0acb9202f Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 12:22:18 +0100 Subject: [PATCH 316/369] test(harness): 2255 seeds beside a holder that cannot read its locations by what it measured, and waits on one that has not looked Every read of the higher holder's location records fails. Beside a holder that measured the folder before its reads failed, the lowest address seeds; beside one that has not looked at the folder since its FluxOS restarted, it waits, and seeds once that holder can read again. Co-Authored-By: Claude Opus 5.5 --- ...locations-keeps-what-it-tells-its-peers.js | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js diff --git a/test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js b/test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js new file mode 100644 index 0000000000..fb3e1866be --- /dev/null +++ b/test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js @@ -0,0 +1,145 @@ +// weight: heavy +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Cold starts of a g: app where the holder at the higher address cannot read its +// locations: every read of its location records fails. Its monitor holds the folder +// out of each pass, and what it tells its peers of the folder stays as it was - a +// read that failed says nothing about what the volume holds. +// +// The lowest address decides the seed. Beside a holder that measured the folder +// before its reads failed, it seeds. Beside one that has not looked at the folder +// since its FluxOS restarted, it waits, and seeds once that holder can read again. +// +// The lowest address is held before it measures until the other holder's reads have +// failed, so the decisions under test are made with the answer that holder gives then. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const COMPONENT = 'data'; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; +const unreadableLocations = (i) => failCommands(i + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); + +describe('a holder that cannot read its locations keeps what it tells its peers', function () { + let env; + let release = null; + dumpLogsOnFailure(() => env); + const client = (i) => env.clients[i]; + + // One app per case, on its own two holders: `lowest` decides, `other` cannot read. + function coldStartCase(tag, { lowest, other }) { + const appName = `e2elocunread${tag}${Date.now()}`; + const watch = watchGComponents(() => env, { appName, components: [COMPONENT], holders: [lowest, other] }); + const folder = watch.folderOf(COMPONENT); + const count = (i, counter, decision) => client(i).getDecisionCount(counter, folder, decision); + return { + folder, + seeded: (i) => count(i, 'syncthing:seedDecision', 'seeded'), + waitedOnLook: (i) => count(i, 'syncthing:seedDecision', 'holderNotLookedAt'), + waitedOnVolume: (i) => count(i, 'syncthing:seedDecision', 'holderVolumeHeldOut'), + passes: (i) => count(i, 'syncthing:folderPass', 'evaluated'), + unread: (i) => count(i, 'syncthing:folderPass', 'locationsUnread'), + + // Placed on both, the lowest held before it measures. + async place() { + expect(subnet.nodeIp(lowest + 1) < subnet.nodeIp(other + 1), 'fixture: the deciding holder is not the lowest address').to.equal(true); + await pushTestApp(appName, 'v1', 'locunread', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: 2, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart([lowest, other], folder); + const from = client(lowest).getLastEventId(); + await client(lowest).holdCheckpoint(BEFORE_HOLDINGS, folder); + await installOnNodes(env, app, [lowest, other]); + await client(lowest).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 300000, { afterId: from }); + await waitFor(async () => await this.passes(other) >= 2, { + timeout: 300000, interval: 2000, label: 'the other holder has measured the folder', + }); + }, + + // The lowest released, until `done` or SETTLE_PASSES of its passes go by. + async decide(done, label) { + const from = await this.passes(lowest); + await client(lowest).releaseCheckpoint(BEFORE_HOLDINGS, folder); + await waitHolding(async () => { + await watch.oneHolderAtMost(); + return await done() || await this.passes(lowest) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label }); + }, + }; + } + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + }); + + afterEach(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + release = null; + await Promise.all((env?.clients ?? []).map((c) => c?.releaseAllCheckpoints().catch(() => {}))); + }); + + after(async function () { + this.timeout(60000); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + it('seeds beside a holder that measured the folder before its reads failed', async function () { + this.timeout(900000); + const c = coldStartCase('measured', { lowest: 1, other: 2 }); + await c.place(); + release = await unreadableLocations(2); + await waitFor(async () => await c.unread(2) >= 1, { + timeout: 120000, interval: 2000, label: 'the other holder cannot read its locations', + }); + + await c.decide(async () => await c.seeded(1) >= 1, 'the lowest address seeds, or the passes go by without it'); + + expect(await c.seeded(1), `the lowest address did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.waitedOnVolume(1), 'the lowest address waited on a holder that only could not read its locations').to.equal(0); + }); + + it('waits on a holder that has not looked at the folder since it restarted, and seeds once it can read again', async function () { + this.timeout(900000); + const c = coldStartCase('notlooked', { lowest: 3, other: 4 }); + await c.place(); + await crashFluxos(client(4).container, { hold: true }); + release = await unreadableLocations(4); + await releaseFluxos(client(4).container); + await waitFor(async () => await c.unread(4) >= 1, { + timeout: 300000, interval: 2000, label: 'the restarted holder cannot read its locations', + }); + + const waitsBefore = await c.waitedOnLook(3); + await c.decide(async () => await c.seeded(3) >= 1, 'the lowest address seeds, or the passes go by without it'); + expect(await c.seeded(3), 'seeded beside a holder that has not looked at the folder').to.equal(0); + expect(await c.waitedOnLook(3) - waitsBefore, 'the lowest address never waited on the holder that has not looked').to.be.at.least(1); + + await release(); + release = null; + await waitFor(async () => await c.seeded(3) >= 1, { + timeout: 300000, interval: 2000, label: 'the lowest address seeds once the other holder can read its locations', + }); + }); +}); From e6045cc0c0377201983e91573dff2860fe04ffcc Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 13:02:23 +0100 Subject: [PATCH 317/369] fix(syncthing): a folder of an app whose spec cannot be read is held out of the cold-start seed Co-Authored-By: Claude Opus 5.5 --- .../services/appMonitoring/folderStates.js | 24 +++++++++-- .../appMonitoring/syncthingMonitor.js | 7 +++ tests/unit/folderStates.test.js | 13 ++++++ tests/unit/syncthingMonitor.test.js | 43 +++++++++++++++++++ 4 files changed, 83 insertions(+), 4 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/folderStates.js b/ZelBack/src/services/appMonitoring/folderStates.js index 3c3c579d80..a8f1c0573b 100644 --- a/ZelBack/src/services/appMonitoring/folderStates.js +++ b/ZelBack/src/services/appMonitoring/folderStates.js @@ -21,8 +21,9 @@ const FolderState = Object.freeze({ // Looked at, with nothing to rank: its holdings could not be read, it has sent // everything it held, or it is a ready standby. NO_CLAIM: 'noClaim', - // Held out of the monitor pass because its volume is not safe to act on, with its - // volume image still on the node: what it holds is unknown. + // Held out of the monitor pass because its volume is not safe to act on or its app's + // specification cannot be read, with its volume image still on the node: what it + // holds is unknown. HELD_OUT: 'heldOut', // Held out of the monitor pass because its volume image is gone: nothing of it is // on the node. @@ -102,14 +103,28 @@ function volumeGone(folderId) { globalState.folderStates.set(folderId, { state: FolderState.VOLUME_GONE }); } +function heldOutUnlessGone(folderId) { + if (globalState.folderStates.get(folderId)?.state === FolderState.VOLUME_GONE) return; + heldOut(folderId); +} + /** * Records a folder the monitor pass found not mounted without asking why. One whose * volume image is known to be gone stays so; any other is held out. * @param {string} folderId */ function notMounted(folderId) { - if (globalState.folderStates.get(folderId)?.state === FolderState.VOLUME_GONE) return; - heldOut(folderId); + heldOutUnlessGone(folderId); +} + +/** + * Records a folder of an app whose specification this node cannot read, which the + * monitor pass does not reach. One whose volume image is known to be gone stays so; + * any other is held out. + * @param {string} folderId + */ +function specUnreadable(folderId) { + heldOutUnlessGone(folderId); } /** @@ -182,5 +197,6 @@ module.exports = { removed, sending, settled, + specUnreadable, volumeGone, }; diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 94eb150762..1f29055418 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -604,6 +604,13 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_AFTER_FOLDER_READ); + // A receive-only folder of an app whose spec cannot be read is one this pass + // does not measure, so what it holds is unknown. Its folders come from + // syncthing's own list, as the sweep below takes them. + allFolders + .filter((folder) => folder.type === 'receiveonly' && ownedByUnreadableApp(folder.id)) + .forEach((folder) => folderStates.specUnreadable(folder.id)); + // Publish which folders this node holds writable, for the peers that ask before // promoting one of their own. Recorded here rather than read on demand: the // answer is a byproduct of a pass the monitor already makes, so serving it costs diff --git a/tests/unit/folderStates.test.js b/tests/unit/folderStates.test.js index e18deeac33..b5ef95e4f1 100644 --- a/tests/unit/folderStates.test.js +++ b/tests/unit/folderStates.test.js @@ -134,6 +134,19 @@ describe('folderStates tests', () => { expect(folderStateMap.get('fluxmeasured')).to.deep.equal({ state: 'heldOut' }); }); + it('keeps a folder whose volume image is gone when its app\'s spec cannot be read, and holds out any other', () => { + folderStates.volumeGone('fluxgone'); + folderStates.measured('fluxmeasured', { bytes: 5, newestModified: 1 }); + folderStates.installed({ name: 'fluxwait', version: 3, containerData: 'g:/appdata' }); + + ['fluxgone', 'fluxmeasured', 'fluxwait', 'fluxnew'].forEach((folderId) => folderStates.specUnreadable(folderId)); + + expect(folderStateMap.get('fluxgone')).to.deep.equal({ state: 'volumeGone' }); + expect(folderStateMap.get('fluxmeasured')).to.deep.equal({ state: 'heldOut' }); + expect(folderStateMap.get('fluxwait')).to.deep.equal({ state: 'heldOut' }); + expect(folderStateMap.get('fluxnew')).to.deep.equal({ state: 'heldOut' }); + }); + it('holds out a folder whose volume image was gone once the mount check finds it on the node', () => { folderStates.volumeGone('fluxa'); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index 354aa637bb..ebdbb2c62c 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -1539,6 +1539,49 @@ describe('syncthingMonitor tests', () => { expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); }); + // A folder of an app whose spec cannot be read is never measured. Left not looked + // at, every peer would wait on this node for as long as the spec stays unreadable, + // even beside a holder that has the app's data. + const specUnreadable = (folders) => { + installedApps([{ name: 'entapp', version: 8, enterprise: 'blob' }]); + appQueryServiceMock.decryptEnterpriseApps.callsFake(async (apps) => ({ readable: [], unreadable: apps, inPlace: [] })); + syncthingServiceMock.getConfigFolders.resolves(folders); + }; + + it('records a receive-only folder of an app whose spec cannot be read as held out, which no peer waits on', async () => { + folderStateMap.set('fluxwebentapp_entapp', { state: 'notLookedAt' }); + specUnreadable([{ id: 'fluxwebentapp_entapp', path: '/apps/fluxwebentapp_entapp', type: 'receiveonly' }]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxwebentapp_entapp')).to.deep.equal({ state: 'heldOut' }); + expect(notLookedAt()).to.deep.equal([]); + }); + + it('records the receive-only folders of an app whose spec cannot be read after a restart, and no other', async () => { + specUnreadable([ + { id: 'fluxwebentapp_entapp', path: '/apps/fluxwebentapp_entapp', type: 'receiveonly' }, + { id: 'fluxdbentapp_entapp', path: '/apps/fluxdbentapp_entapp', type: 'sendreceive' }, + { id: 'fluxwebother_other', path: '/apps/fluxwebother_other', type: 'receiveonly' }, + ]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect([...folderStateMap.entries()]).to.deep.equal([['fluxwebentapp_entapp', { state: 'heldOut' }]]); + }); + + it('keeps a folder of an app whose spec cannot be read as volume gone when its volume image is gone', async () => { + folderStateMap.set('fluxwebentapp_entapp', { state: 'volumeGone' }); + specUnreadable([{ id: 'fluxwebentapp_entapp', path: '/apps/fluxwebentapp_entapp', type: 'receiveonly' }]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxwebentapp_entapp')).to.deep.equal({ state: 'volumeGone' }); + }); + // A read that failed says nothing about what the volume holds. const locationsUnreadable = () => { installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); From 9f944e8388b62f1b17001c180ff2eed2a7f5456d Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 13:05:11 +0100 Subject: [PATCH 318/369] test(harness): 2256 leaves a holder that cannot decrypt an app's spec out of the cold-start seed Co-Authored-By: Claude Opus 5.5 --- ...pt-an-apps-spec-is-left-out-of-the-seed.js | 174 ++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js diff --git a/test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js b/test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js new file mode 100644 index 0000000000..1756dea0fa --- /dev/null +++ b/test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js @@ -0,0 +1,174 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart, pinHolding } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { failRpc, clearRpcFailures } from '../framework/daemon-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { buildSeedableEnterpriseApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Cold starts of an enterprise g: app where a holder comes back from a restart unable +// to decrypt the app's spec: the benchmark refuses it the key. Its monitor pass does +// not reach the app's folder, so what it holds is unknown. +// +// It is left out of the choice: a holder with data seeds, and while no other holder +// has data nobody seeds until it can decrypt again. Once it can, it measures the +// folder and the seed goes ahead. +// +// Every other holder is held before it measures until each broken holder answers its +// peers again, so the decisions under test are made with the broken holders' answers. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const COMPONENT = 'data'; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; +const DATA_BYTES = 5821604997; +const DECRYPT = 'decryptrsamessage'; +const nodeIp = (i) => subnet.nodeIp(i + 1); + +describe('a holder that cannot decrypt an app\'s spec is left out of the seed', function () { + let env; + dumpLogsOnFailure(() => env); + const client = (i) => env.clients[i]; + + // One app per case, on its own two holders, decided from a cold start. + function coldStartCase(tag, { lowest, other, broken, dataOn }) { + const holders = [lowest, other]; + const appName = `e2enodecrypt${tag}${Date.now()}`; + const watch = watchGComponents(() => env, { appName, components: [COMPONENT], holders }); + const folder = watch.folderOf(COMPONENT); + const count = (i, counter, decision) => client(i).getDecisionCount(counter, folder, decision); + const decider = holders.find((i) => i !== broken); + const seeded = (i) => count(i, 'syncthing:seedDecision', 'seeded'); + + return { + folder, + decider, + oneHolderAtMost: watch.oneHolderAtMost, + seeded, + cannotSeed: (i) => count(i, 'syncthing:seedDecision', 'holderCannotSeed'), + waitedOnVolume: (i) => count(i, 'syncthing:seedDecision', 'holderVolumeHeldOut'), + passes: (i) => count(i, 'syncthing:folderPass', 'evaluated'), + seeders: async () => (await Promise.all(holders.map(seeded))).map((n, k) => (n > 0 ? holders[k] : null)).filter((i) => i !== null), + + async start() { + expect(nodeIp(lowest) < nodeIp(other), 'fixture: the lowest holder is not the lowest address').to.equal(true); + await pushTestApp(appName, 'v1', 'nodecrypt', { user: APP_UID }); + const app = await buildSeedableEnterpriseApp({ + env, name: appName, instances: holders.length, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart(holders, folder); + if (dataOn !== undefined) await pinHolding(dataOn, folder, DATA_BYTES); + const from = client(decider).getLastEventId(); + await client(decider).holdCheckpoint(BEFORE_HOLDINGS, folder); + await installOnNodes(env, app, holders); + await client(decider).waitForEvent( + 'checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, + 300000, + { afterId: from }, + ); + + await crashFluxos(client(broken).container, { hold: true }); + await failRpc(nodeIp(broken), DECRYPT, { times: 1000000 }); + await releaseFluxos(client(broken).container); + await waitFor(async () => (await client(broken).getPromotedFolders())?.data?.ready === true, { + timeout: 300000, interval: 2000, label: `node ${broken} answers its peers after its restart`, + }); + await client(decider).releaseCheckpoint(BEFORE_HOLDINGS, folder); + }, + + // Until `done`, or SETTLE_PASSES folder passes go by on the decider; never more + // than one holder running the app. + async settle(done, label) { + const from = await this.passes(decider); + await waitHolding(async () => { + await watch.oneHolderAtMost(); + return await done() || await this.passes(decider) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label }); + }, + }; + } + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 4, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + }); + + after(async function () { + this.timeout(60000); + await Promise.all((env?.clients ?? []).map((c) => c?.releaseAllCheckpoints().catch(() => {}))); + await Promise.all([0, 1, 2, 3].map((i) => clearRpcFailures(nodeIp(i)).catch(() => {}))); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + describe('the lowest address cannot decrypt the spec, and another holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('data', { + lowest: 0, other: 1, broken: 0, dataOn: 1, + }))); + + it('seeds on the holder with data', async function () { + this.timeout(900000); + await c.start(); + await c.settle(async () => await c.seeded(1) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeded(1), `the holder with data did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.seeded(0), 'the holder that cannot decrypt seeded').to.equal(0); + }); + + it('takes the holder back as a standby once it can decrypt', async function () { + this.timeout(600000); + const from = await c.passes(0); + await clearRpcFailures(nodeIp(0)); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return await c.passes(0) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label: `the holder that can decrypt again passes its folder ${SETTLE_PASSES} times` }); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([1]); + }); + }); + + describe('a holder that cannot decrypt the spec may hold the data, and the lowest address has none', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('wait', { + lowest: 2, other: 3, broken: 3, dataOn: 3, + }))); + + it('seeds nothing while that holder cannot decrypt', async function () { + this.timeout(900000); + await c.start(); + await c.settle(async () => await c.seeded(2) >= 1, 'the lowest address seeds, or the passes go by without it'); + + expect(await c.seeded(2), 'an empty copy seeded over a folder that may hold the app\'s data').to.equal(0); + expect(await c.waitedOnVolume(2), 'the lowest address never waited on the holder that cannot decrypt').to.be.at.least(1); + expect(await c.seeded(3), 'the holder that cannot decrypt seeded').to.equal(0); + }); + + it('seeds on the holder with data once it can decrypt', async function () { + this.timeout(600000); + await clearRpcFailures(nodeIp(3)); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return (await c.seeders()).length >= 1; + }, { timeout: 300000, interval: 2000, label: 'a holder seeds once the spec can be decrypted' }); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([3]); + }); + }); +}); From d2d8e14e00db56254d6e9cbfcaa5a0ef8207c025 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:11:09 +0100 Subject: [PATCH 319/369] fix(syncthing): a revert of a standby's changes goes ahead only on the folder checked, read and sent while its config is held against every write Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 68 +++++++++++---- .../appMonitoring/syncthingFolderWrites.js | 1 + tests/unit/localChangeCheck.test.js | 86 ++++++++++++++++++- .../unit/syncthingFolderStateMachine.test.js | 3 + 4 files changed, 139 insertions(+), 19 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js index b192285fc9..2ac2b400fd 100644 --- a/ZelBack/src/services/appMonitoring/localChangeCheck.js +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -3,6 +3,7 @@ const log = require('../../lib/log'); const syncthingService = require('../syncthingService'); const messageHelper = require('../messageHelper'); const fluxEventBus = require('../utils/fluxEventBus'); +const syncthingFolderWrites = require('./syncthingFolderWrites'); // Whether a receive-only folder's local changes include the only copy of data another // device wrote, decided off the monitor pass. @@ -21,10 +22,14 @@ const fluxEventBus = require('../utils/fluxEventBus'); // neither, so an answer stands while the primary keeps writing. The pass acts on an // answer only while the folder still carries its stamp. // -// A REVERT IS OF THE CHANGES CHECKED, AND NO OTHERS. Counts can stay the same while -// the changes do not, so the pass does not revert itself: it asks for a revert here, -// in the same queue as the checks, and the revert reads the folder's changes again -// and goes ahead only while they are exactly the ones the check examined. +// A REVERT IS OF THE CHANGES CHECKED, AND NO OTHERS, ON THE FOLDER CHECKED. Counts can +// stay the same while the changes do not, so the pass does not revert itself: it asks +// for a revert here, in the same queue as the checks. The revert reads the folder's +// changes again and goes ahead only while they are exactly the ones the check +// examined, and the folder still carries the check's stamp. The stamp is read and the +// revert sent while the folder's config is held against every write +// (syncthingFolderWrites), so no device leaves the folder between the two; a write +// that arrives once syncthing has the revert waits inside syncthing for its work. // syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. const RECEIVE_ONLY_CHANGE_FLAG = 8; @@ -46,12 +51,13 @@ let running = null; /** * The folder as syncthing describes it, in the form an answer is stamped with. - * @param {object} status The folder's db/status: its counts of receive-only changes. + * @param {object} status The folder's db/status: its counts of receive-only changes, + * each absent one read as none. * @param {Array<{deviceID: string}>} devices The devices the folder is shared with. * @returns {string} */ function stampOf(status, devices) { - const changes = CHANGE_COUNTS.map((count) => status?.[count] ?? null); + const changes = CHANGE_COUNTS.map((count) => status?.[count] ?? 0); const shared = devices.map((device) => device.deviceID).sort(); return JSON.stringify({ changes, shared }); } @@ -166,10 +172,26 @@ async function runCheck(check) { fluxEventBus.publish('syncthing:localChangesChecked', { folder: check.folderId, onlyCopy: Boolean(found.onlyCopy), madeHere: found.madeHere }); } +/** + * The folder as syncthing describes it now, in the form an answer is stamped with. + * @param {string} folderId + * @returns {Promise} null when it could not be read + */ +async function currentStamp(folderId) { + try { + const [status, folder] = await Promise.all([syncthingService.getDbStatus(folderId), syncthingService.getConfigFolders(folderId)]); + return Array.isArray(folder?.devices) ? stampOf(status, folder.devices) : null; + } catch (error) { + log.warn(`localChangeCheck - ${folderId}: ${error.message}`); + return null; + } +} + /** * Reverts a folder's local changes if they are exactly the ones its answer examined, - * and the entry has not been replaced. Otherwise the answer is dropped, so the pass - * asks for the folder to be checked again. + * the folder still carries the answer's stamp, and the entry has not been replaced. + * Otherwise the answer is dropped, so the pass asks for the folder to be checked + * again; a folder that could not be read keeps it. * @param {{folderId: string, entry: object, answer: object, isCurrent: Function, files: number, directories: number}} job */ async function runRevert(job) { @@ -183,16 +205,26 @@ async function runRevert(job) { log.info(`localChangeCheck - ${folderId}: its local changes are not the ones checked; checking them again before any revert`); return; } - log.warn(`localChangeCheck - ${folderId} is synced but the receive-only folder has local changes (${job.files} files, ${job.directories} directories); reverting to the cluster's version`); - try { - // dataOrThrow: dbRevert answers in-band; without it a failed revert reads as reverted - messageHelper.dataOrThrow(await syncthingService.dbRevert(folderId)); - // eslint-disable-next-line no-param-reassign - delete entry.localChangeCheck; - fluxEventBus.publish('syncthing:localChangesReverted', { folder: folderId, files: job.files, directories: job.directories }); - } catch (error) { - log.error(`localChangeCheck - revert of local changes for ${folderId} failed: ${error.message}`); - } + await syncthingFolderWrites.exclusive([folderId], async () => { + const stamp = await currentStamp(folderId); + if (stamp === null || !job.isCurrent() || entry.localChangeCheck !== answer) return; + if (stamp !== answer.stamp) { + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + log.info(`localChangeCheck - ${folderId}: the folder is not the one checked; checking it again before any revert`); + return; + } + log.warn(`localChangeCheck - ${folderId} is synced but the receive-only folder has local changes (${job.files} files, ${job.directories} directories); reverting to the cluster's version`); + try { + // dataOrThrow: dbRevert answers in-band; without it a failed revert reads as reverted + messageHelper.dataOrThrow(await syncthingService.dbRevert(folderId)); + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + fluxEventBus.publish('syncthing:localChangesReverted', { folder: folderId, files: job.files, directories: job.directories }); + } catch (error) { + log.error(`localChangeCheck - revert of local changes for ${folderId} failed: ${error.message}`); + } + }); } async function runWaitingJobs() { diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js b/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js index b70d069bc3..89edeed48a 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js @@ -491,6 +491,7 @@ async function changeSyncthingFolderType(folderId, folderType, { module.exports = { FOLDER_TYPE_SETTLE_MS, changeSyncthingFolderType, + exclusive, folderConfig, pauseAllFolders, putFolders, diff --git a/tests/unit/localChangeCheck.test.js b/tests/unit/localChangeCheck.test.js index d20ed65c92..fb6ef2a48b 100644 --- a/tests/unit/localChangeCheck.test.js +++ b/tests/unit/localChangeCheck.test.js @@ -2,12 +2,15 @@ const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); +const syncthingFolderWrites = require('../../ZelBack/src/services/appMonitoring/syncthingFolderWrites'); const syncthingServiceMock = { getDeviceId: sinon.stub(), getDbLocalChanged: sinon.stub(), getDbFile: sinon.stub(), dbRevert: sinon.stub(), + getDbStatus: sinon.stub(), + getConfigFolders: sinon.stub(), }; const localChangeCheck = proxyquire('../../ZelBack/src/services/appMonitoring/localChangeCheck', { '../syncthingService': syncthingServiceMock, @@ -27,6 +30,10 @@ function held(answer) { return { promise, release }; } +// The folder a revert's check was made against, as syncthing describes it. +const FOLDER_STATUS = { receiveOnlyChangedFiles: 1, receiveOnlyChangedBytes: 10 }; +const FOLDER_DEVICES = [{ deviceID: 'LOCAL-DEVICE' }, { deviceID: 'PRIMARY-DEVICE' }]; + describe('localChangeCheck tests', () => { beforeEach(() => { syncthingServiceMock.getDeviceId.reset(); @@ -37,6 +44,10 @@ describe('localChangeCheck tests', () => { syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 } }); syncthingServiceMock.dbRevert.reset(); syncthingServiceMock.dbRevert.resolves({ status: 'success', data: {} }); + syncthingServiceMock.getDbStatus.reset(); + syncthingServiceMock.getDbStatus.resolves(FOLDER_STATUS); + syncthingServiceMock.getConfigFolders.reset(); + syncthingServiceMock.getConfigFolders.resolves({ devices: FOLDER_DEVICES }); }); // An answer with no only copy, as runCheck leaves it. @@ -219,7 +230,7 @@ describe('localChangeCheck tests', () => { async function checked(folderId = 'fluxr_r') { syncthingServiceMock.getDbLocalChanged.withArgs(folderId).resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); const entry = {}; - localChangeCheck.request(folderId, entry, 's1', () => true); + localChangeCheck.request(folderId, entry, localChangeCheck.stampOf(FOLDER_STATUS, FOLDER_DEVICES), () => true); await settled(); return entry; } @@ -307,6 +318,79 @@ describe('localChangeCheck tests', () => { sinon.assert.calledOnce(syncthingServiceMock.dbRevert); }); + it('does not revert a folder a device has left since the check, and drops the answer', async () => { + const entry = await checked(); + syncthingServiceMock.getConfigFolders.resolves({ devices: [{ deviceID: 'LOCAL-DEVICE' }] }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('does not revert a folder whose counts of changes moved since the check, and drops the answer', async () => { + const entry = await checked(); + syncthingServiceMock.getDbStatus.resolves({ ...FOLDER_STATUS, receiveOnlyChangedBytes: 11 }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('does not revert while the folder cannot be read, and keeps the answer', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + syncthingServiceMock.getConfigFolders.rejects(new Error('syncthing did not answer')); + + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry.localChangeCheck).to.equal(answer); + }); + + // The order a monitor pass makes possible: the revert is asked for, and the pass's + // write of the folder's devices, which drops the primary, holds the folder first. + it('waits for a write to the folder that holds it, and does not revert once that write has dropped a device', async () => { + const entry = await checked(); + const write = held(); + const writing = syncthingFolderWrites.exclusive(['fluxr_r'], async () => { + await write.promise; + syncthingServiceMock.getConfigFolders.resolves({ devices: [{ deviceID: 'LOCAL-DEVICE' }] }); + }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + sinon.assert.notCalled(syncthingServiceMock.getConfigFolders); + write.release(); + await writing; + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('holds the folder against every write until the revert is sent', async () => { + const entry = await checked(); + const sent = held({ status: 'success', data: {} }); + syncthingServiceMock.dbRevert.returns(sent.promise); + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + + let wrote = false; + const writing = syncthingFolderWrites.exclusive(['fluxr_r'], async () => { wrote = true; }); + await settled(); + expect(wrote, 'a write to the folder went in while its revert was being sent').to.equal(false); + sent.release(); + await writing; + + expect(wrote).to.equal(true); + }); + it('does not start again a revert already running on the same answer', async () => { const entry = await checked(); const again = held({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 072fd6a12f..cbaf0a8423 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -2353,6 +2353,7 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder.type).to.equal('receiveonly'); @@ -3100,6 +3101,7 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + await checksSettled(); sinon.assert.calledWith(syncthingServiceMock.dbRevert, 'test-app'); }); @@ -3348,6 +3350,7 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder.type).to.equal('receiveonly'); From fd6d9072393da5b2f97b1acd8e3cace923ebf483 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 15:14:08 +0100 Subject: [PATCH 320/369] test(harness): 3317 keeps a standby's change whose revert was held while the primary left the folder Co-Authored-By: Claude Opus 5.5 --- ...changes-while-the-primary-keeps-writing.js | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js index 301353327a..af46c31370 100644 --- a/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js +++ b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js @@ -8,6 +8,10 @@ * folder's changes again, while a further change lands, does not go ahead until the * folder is checked again. * + * And it is of the folder checked: one held while the primary's location expires and + * the primary leaves the standby's folder, which makes the standby's change to the + * primary's file the only copy, does not go ahead, and that change is kept. + * * A = the standby, B = the primary, C = holds nothing. */ import { @@ -17,8 +21,11 @@ import { expect } from 'chai'; import { placeSilentPrimaryApp, A, B, } from '../framework/silent-primary.js'; -import { execInContainer } from '../framework/container.js'; -import { readPath, scanFolder } from '../framework/syncthing-real.js'; +import { execInContainer, crashFluxos } from '../framework/container.js'; +import { dbClient } from '../framework/db-client.js'; +import { + readPath, scanFolder, getFolderConfig, getDeviceId, +} from '../framework/syncthing-real.js'; import { waitFor } from '../framework/wait.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -138,4 +145,42 @@ describe('a standby reverts its local changes while the primary keeps writing', timeout: 120000, interval: 2000, label: 'both strays are gone from the standby', }); }); + + // Last: the primary is fenced from here on. + it('keeps a change to the primary\'s file whose revert was held while the primary left the folder', async function () { + this.timeout(REVERT_MS + 600000); + writing = false; + await writer; + const kept = `${root}/appdata/kept`; + await sh(B, `printf primary > ${kept}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), kept)) === 'primary', { + timeout: 120000, interval: 2000, label: 'the primary\'s file reached the standby', + }); + + await client(A).holdCheckpoint(BEFORE_REVERT, folder); + const from = client(A).getLastEventId(); + step('the standby changes the primary\'s file, and its revert is held'); + await sh(A, `printf standby > ${kept}`); + await scanFolder(client(A), folder); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_REVERT && d.key === folder, REVERT_MS, { afterId: from }); + + step('the primary stops announcing, its location expires, and it leaves the standby\'s folder'); + const primaryDevice = await getDeviceId(client(B)); + await crashFluxos(client(B).container, { hold: true }); + const expired = await dbClient(A + 1).expireAppLocation(appName, client(B).ip); + expect(expired, 'fixture: the standby held no location for the primary, so none expired').to.be.at.least(1); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []).some((d) => d.deviceID === primaryDevice), { + timeout: 300000, interval: 2000, label: 'the primary left the standby\'s folder', + }); + + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_REVERT, folder); + const onlyCopy = await client(A).waitForEvent('syncthing:localChangesChecked', (d) => d.folder === folder && d.onlyCopy, REVERT_MS, { afterId: releasedAt }) + .catch(() => null); + + expect(eventsSince('syncthing:localChangesReverted', releasedAt), 'reverted the standby\'s change once it was the only copy').to.deep.equal([]); + expect(await readPath(client(A), kept), 'the standby\'s change to the primary\'s file').to.equal('standby'); + expect(onlyCopy, 'the standby never checked the folder again and found its change the only copy').to.not.equal(null); + }); }); From 29008eff6e2c064b8a255d03c93ea4bad2ba5584 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 19:48:10 +0100 Subject: [PATCH 321/369] test(harness): 3320 reverts a standby's change with the primary still in the folder, and removes the primary from it after The primary's device becomes one the monitor deletes while the standby's revert of a change to the primary's file is held after its last read, inside the folder's lock: its location expires and another device is configured under its name. The standby's file is kept, the primary's device stays in the folder until the revert is sent, and it leaves the folder through the folder's write after. syncthing:beforeLocalChangeRevertSent holds a revert that has found the folder the one it checked and has not sent the revert. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/localChangeCheck.js | 1 + ZelBack/src/services/utils/fluxEventBus.js | 4 + ...ot-removed-from-a-folder-a-revert-holds.js | 125 ++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js index 2ac2b400fd..9a22a73e00 100644 --- a/ZelBack/src/services/appMonitoring/localChangeCheck.js +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -214,6 +214,7 @@ async function runRevert(job) { log.info(`localChangeCheck - ${folderId}: the folder is not the one checked; checking it again before any revert`); return; } + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT_SENT, folderId); log.warn(`localChangeCheck - ${folderId} is synced but the receive-only folder has local changes (${job.files} files, ${job.directories} directories); reverting to the cluster's version`); try { // dataOrThrow: dbRevert answers in-band; without it a failed revert reads as reverted diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index eb356db371..b3d3451ac3 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -77,6 +77,10 @@ const Checkpoint = Object.freeze({ // A revert of a receive-only folder's local changes has not yet read whether they // are still the ones checked. Keyed by folder id. SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT: 'syncthing:beforeLocalChangeRevert', + // A revert of a receive-only folder's local changes holds the folder's config, + // has found the folder the one checked, and has not yet sent the revert. Keyed + // by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT_SENT: 'syncthing:beforeLocalChangeRevertSent', // The syncthing monitor has reached a receive-only folder and not yet measured // what this node holds in it. Keyed by folder id. SYNCTHING_BEFORE_HOLDINGS: 'syncthing:beforeHoldings', diff --git a/test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js b/test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js new file mode 100644 index 0000000000..fd9df47cb4 --- /dev/null +++ b/test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js @@ -0,0 +1,125 @@ +// weight: heavy +/* + * Syncthing drops a deleted device from every folder that lists it, outside + * FluxOS's writes to those folders. A revert of a standby's changes holds the + * folder against those writes from the moment it finds the folder the one it + * checked until the revert is sent, so a device the monitor no longer wants + * stays configured while any folder lists it, and leaves the folder through the + * folder's own write. + * + * The primary's device becomes one the monitor deletes while the standby's revert + * of a change to the primary's file is held after its last read: the primary + * stops announcing, its location expires, and another device is configured under + * its name. The standby's change is reverted with the primary still in the + * folder, so it is not deleted, and the primary leaves the folder only once the + * revert has been sent. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { execInContainer, crashFluxos } from '../framework/container.js'; +import { dbClient } from '../framework/db-client.js'; +import { + readPath, scanFolder, getFolderConfig, getFolderStatus, getDeviceId, getConfiguredDevices, addDevice, +} from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const REVERT_SENT = 'syncthing:beforeLocalChangeRevertSent'; +const REVERT_MS = 5 * 60 * 1000; + +function step(what) { + console.log(`# 3320 ${new Date().toISOString()} ${what}`); +} + +describe('a device is not removed from a folder a revert holds', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2eheldrevert${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + const folderDevices = async () => ((await getFolderConfig(client(A), folder))?.devices ?? []).map((d) => d.deviceID); + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + }); + + afterEach(async function () { + await client(A)?.releaseAllCheckpoints().catch(() => {}); + }); + + after(async function () { + this.timeout(60000); + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('reverts the standby\'s change while the primary is still in the folder, and removes the primary from it after', async function () { + this.timeout(REVERT_MS + 600000); + const held = `${root}/appdata/held`; + await sh(B, `printf primary > ${held}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), held)) === 'primary', { + timeout: 120000, interval: 2000, label: 'the primary\'s file reached the standby', + }); + const primaryDevice = await getDeviceId(client(B)); + const primaryName = (await getConfiguredDevices(client(A))).find((d) => d.deviceID === primaryDevice)?.name; + expect(primaryName, 'fixture: the standby has no device configured for the primary').to.be.a('string'); + + await client(A).holdCheckpoint(REVERT_SENT, folder); + const from = client(A).getLastEventId(); + step('the standby changes the primary\'s file, and its revert is held after its last read'); + await sh(A, `printf standby > ${held}`); + await scanFolder(client(A), folder); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === REVERT_SENT && d.key === folder, REVERT_MS, { afterId: from }); + + step('the primary stops announcing, its location expires, and another device is configured under its name'); + await crashFluxos(client(B).container, { hold: true }); + const expired = await dbClient(A + 1).expireAppLocation(appName, client(B).ip); + expect(expired, 'fixture: the standby held no location for the primary, so none expired').to.be.at.least(1); + await addDevice(client(A), { + deviceID: await getDeviceId(client(C)), + name: primaryName, + address: `tcp://${client(B).ip}:16129`, + }); + const passFrom = client(A).getLastEventId(); + await client(A).waitForEvent('syncthing:passComplete', () => true, 120000, { afterId: passFrom }); + const configuredWhileHeld = (await getConfiguredDevices(client(A))).some((d) => d.deviceID === primaryDevice); + const inFolderWhileHeld = (await folderDevices()).includes(primaryDevice); + + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(REVERT_SENT, folder); + await client(A).waitForEvent('syncthing:localChangesReverted', (d) => d.folder === folder, REVERT_MS, { afterId: releasedAt }) + .catch(() => { + throw new Error('the standby did not revert its change once the revert was released'); + }); + await waitFor(async () => !(await folderDevices()).includes(primaryDevice), { + timeout: 300000, interval: 2000, label: 'the primary left the standby\'s folder', + }); + await waitFor(async () => (await getFolderStatus(client(A), folder))?.state === 'idle', { + timeout: 120000, interval: 2000, label: 'the standby\'s folder is idle', + }); + + // Reverted while the primary was in the folder, the file is the primary's + // version once pulled, or the standby's while the primary cannot be reached; + // reverted with no other device in the folder, it is deleted. + expect(await readPath(client(A), held), 'the standby\'s file after its revert').to.be.oneOf(['primary', 'standby']); + expect(configuredWhileHeld, 'the primary\'s device was deleted while a folder listed it').to.equal(true); + expect(inFolderWhileHeld, 'the primary left the folder while its revert was held').to.equal(true); + }); +}); From fae8965096373b962b099a2bcfb1d90064af8ef1 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 19:48:11 +0100 Subject: [PATCH 322/369] fix(syncthing): a device any folder lists is not deleted, so it never leaves a folder outside the folder's writes Syncthing drops a deleted device from every folder that lists it, without a write to the folder, and a revert of a standby's changes holds a folder's devices fixed only against FluxOS's writes to it. A device the monitor no longer wants leaves its folders through those writes, and the device sweep deletes it on the first pass that finds it in none. Co-Authored-By: Claude Opus 5.5 --- .../appMonitoring/syncthingMonitor.js | 7 ++++ tests/unit/syncthingMonitor.test.js | 35 +++++++++++++++++-- 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 1f29055418..2ebc550c7e 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -929,12 +929,19 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // A device seen within UNUSED_DEVICE_KEEP_MS stays too, unless another device // is configured under its name - then it is not the device at that address - // and every device stays while when each was last seen cannot be read. + // A device any folder lists stays as well: syncthing drops a deleted device + // from every folder that lists it, outside the folder's writes here, and a + // revert of a standby's changes holds the folder's devices fixed only against + // those writes. It leaves its folders through the folder writes below, and is + // deleted by the first pass that finds it in none. const deviceStats = unsafeFolderIds.size > 0 ? null : await syncthingService.getDeviceStats().catch(() => null); const seenRecently = (deviceID) => Date.now() - Date.parse(deviceStats?.[deviceID]?.lastSeen) < UNUSED_DEVICE_KEEP_MS; const nameHeldByAnother = (device) => Boolean(device.name) && allDevices.some((other) => other.deviceID !== device.deviceID && other.name === device.name); + const listedInAFolder = new Set(allFolders.flatMap((folder) => (folder.devices ?? []).map(({ deviceID }) => deviceID))); const nonUsedDevices = !deviceStats ? [] : allDevices.filter( (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId + && !listedInAFolder.has(syncthingDevice.deviceID) && (!seenRecently(syncthingDevice.deviceID) || nameHeldByAnother(syncthingDevice)), ); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index ebdbb2c62c..e5104b2f3f 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -2018,7 +2018,9 @@ describe('syncthingMonitor tests', () => { // folder drops syncthing's whole record of the app: the index, the peer // devices and, on the next pass, any mount-safety demotion standing on it. describe('a peer no location names any more', () => { - function folderSharedWithPeer() { + // The peer's device is configured; `peerInFolder` is whether syncthing's + // folder still lists it, as it does until a folder write takes it out. + function folderSharedWithPeer({ peerInFolder = false } = {}) { mockState.backupInProgress = []; mockState.restoreInProgress = []; syncthingMonitorHelpersMock.buildDeviceConfiguration.callsFake(async () => [{ deviceID: 'DEVICE-ID' }]); @@ -2028,7 +2030,7 @@ describe('syncthingMonitor tests', () => { fluxNetworkHelperMock.getLocalSocketAddress.resolves('10.0.0.1:16127'); mockInstalledAppsFn.resolves({ status: 'success', data: [{ name: 'testapp', version: 3, containerData: 'g:/appdata' }] }); syncthingServiceMock.getConfigFolders.resolves([ - { id: 'testapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }, { deviceID: 'PEER-DEVICE' }] }, + { id: 'testapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }, ...(peerInFolder ? [{ deviceID: 'PEER-DEVICE' }] : [])] }, ]); syncthingServiceMock.getConfigDevices.resolves([{ deviceID: 'PEER-DEVICE' }]); syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); @@ -2045,7 +2047,7 @@ describe('syncthingMonitor tests', () => { // A node whose FluxOS has stopped answering stops announcing while its // syncthing and its app carry on; once its location expires its writes reach // no holder through this one. - it('removes its device though it is connected and shares the folder', async () => { + it('removes its device though it is connected, once no folder lists it', async () => { folderSharedWithPeer(); syncthingServiceMock.getDbCompletion.withArgs({ folder: 'testapp', device: 'PEER-DEVICE' }).resolves({ remoteState: 'valid' }); @@ -2055,8 +2057,35 @@ describe('syncthingMonitor tests', () => { sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); }); + // Syncthing drops a deleted device from every folder that lists it, outside + // the folder's writes, which a revert of a standby's changes holds off. + it('keeps its device while a folder still lists it', async () => { + folderSharedWithPeer({ peerInFolder: true }); + syncthingServiceMock.getDbCompletion.withArgs({ folder: 'testapp', device: 'PEER-DEVICE' }).resolves({ remoteState: 'valid' }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledOnce(syncthingServiceMock.getDeviceStats); + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + // Its connection record is what proves it gone to a standby that remembers it as // the app's primary. + it('keeps its device while a folder this pass deletes still lists it', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getConfigFolders.resolves([ + { id: 'testapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }] }, + { id: 'goneapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }, { deviceID: 'PEER-DEVICE' }] }, + ]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledWithExactly(syncthingServiceMock.adjustConfigFolders, 'delete', undefined, 'goneapp'); + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + it('keeps the entry of a device it dropped while syncthing saw it within a day', async () => { folderSharedWithPeer(); syncthingServiceMock.getDeviceStats.resolves({ 'PEER-DEVICE': { lastSeen: new Date(Date.now() - 60 * 60 * 1000).toISOString() } }); From 9af013b6345b421f4f32413a39a6585a78c93bac Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 21:03:19 +0100 Subject: [PATCH 323/369] test(syncthing): the connection-record comment sits on the test it describes Co-Authored-By: Claude Opus 5.5 --- tests/unit/syncthingMonitor.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index e5104b2f3f..a01d3ed9ca 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -2070,8 +2070,6 @@ describe('syncthingMonitor tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); }); - // Its connection record is what proves it gone to a standby that remembers it as - // the app's primary. it('keeps its device while a folder this pass deletes still lists it', async () => { folderSharedWithPeer(); syncthingServiceMock.getConfigFolders.resolves([ @@ -2086,6 +2084,8 @@ describe('syncthingMonitor tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); }); + // Its connection record is what proves it gone to a standby that remembers it as + // the app's primary. it('keeps the entry of a device it dropped while syncthing saw it within a day', async () => { folderSharedWithPeer(); syncthingServiceMock.getDeviceStats.resolves({ 'PEER-DEVICE': { lastSeen: new Date(Date.now() - 60 * 60 * 1000).toISOString() } }); From 5e867561f6111a6ed5c2df1c7b0b8e96c653617b Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 07:29:01 +0100 Subject: [PATCH 324/369] fix(apps): an apprunning message is stored one entry at a time A version 2 apprunning message is judged per entry: an entry with a string name and hash is stored, any other is refused and logged, and the rest of the message is unaffected. Live gossip and peer sync apply the same rule, so neither writes a location row without a hash. A message whose every entry is refused stores nothing and is not relayed. It is never read as an empty snapshot, which deletes the sender's rows. A message the store refuses outright is logged where it arrived. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMessaging/messageStore.js | 37 +++- ZelBack/src/services/fluxCommunication.js | 4 + .../src/services/utils/appRunningEntries.js | 25 +++ tests/unit/fluxCommunication.test.js | 20 ++ .../messageStoreAppRunningEntries.test.js | 176 ++++++++++++++++++ 5 files changed, 253 insertions(+), 9 deletions(-) create mode 100644 ZelBack/src/services/utils/appRunningEntries.js create mode 100644 tests/unit/messageStoreAppRunningEntries.test.js diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index d75af73507..c34742a067 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -10,6 +10,7 @@ const { socketAddressesMatch } = require('../utils/socketAddressUtils'); // const advancedWorkflows = require('../appLifecycle/advancedWorkflows'); // Moved to dynamic require to avoid circular dependency const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const globalState = require('../utils/globalState'); +const { appRunningEntries } = require('../utils/appRunningEntries'); const { globalAppsMessages, globalAppsTempMessages, @@ -289,9 +290,27 @@ function runningSinceOf(isNewer, runningSince, moved) { } /** - * Store app running message + * Log the entries of one apprunning message that were not stored. + * @param {string} caller The function that refused them. + * @param {object} message The message's data. + * @param {Array<*>} refused The refused entries. + */ +function logRefusedEntries(caller, message, refused) { + if (!refused.length) return; + const shown = refused.slice(0, 5).map((app) => JSON.stringify(app)?.slice(0, 200)).join(', '); + log.warn(`${caller} - ${refused.length} app entries from ${message.ip} at ${message.broadcastedAt} have no string name and hash and are not stored: ${shown}`); +} + +/** + * Store app running message. + * + * A version 2 message is stored entry by entry: the entries with a name and a + * hash are written and the rest are refused, logged, and leave the others alone. + * A message whose every entry is refused stores nothing and is not relayed. It + * is never read as an empty snapshot, which deletes the sender's rows: only a + * message that names no apps at all says that. * @param {object} message - Message to store - * @returns {Promise} Whether message should be rebroadcast or Error if invalid + * @returns {Promise<{stored: boolean, rebroadcast: boolean}|Error>} What was stored, or Error when the message itself is malformed */ async function storeAppRunningMessage(message) { /* message object @@ -334,13 +353,12 @@ async function storeAppRunningMessage(message) { if (message.apps.length > config.fluxapps.maxAppsPerNode) { return new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode'); } - for (let i = 0; i < message.apps.length; i += 1) { - const app = message.apps[i]; - appsMessages.push(app); - if (typeof app.hash !== 'string' || typeof app.name !== 'string') { - return new Error('Invalid Flux App Running v2 message for storing'); - } + const { stored, refused } = appRunningEntries(message.apps); + logRefusedEntries('storeAppRunningMessage', message, refused); + if (message.apps.length && !stored.length) { + return { stored: false, rebroadcast: false }; } + appsMessages.push(...stored); } if (message.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) { @@ -795,7 +813,8 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { if (validTill < Date.now()) continue; const ip = addresses[index]; - const apps = data.version === 2 ? (data.apps || []) : [{ name: data.name, hash: data.hash }]; + const { stored: apps, refused } = appRunningEntries(data.version === 2 ? (data.apps || []) : [{ name: data.name, hash: data.hash }]); + logRefusedEntries('storeBatchAppRunningMessages', data, refused); const incomingDate = new Date(data.broadcastedAt); const incomingExpiry = new Date(validTill); const isNewer = { $gt: [incomingDate, { $ifNull: ['$broadcastedAt', new Date(0)] }] }; diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 2de28294c3..37415f7c89 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -460,6 +460,10 @@ async function handleAppRunningMessage(message, fromIP, port) { try { await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: message }); const result = await messageStore.storeAppRunningMessage(message.data); + if (result instanceof Error) { + log.warn(`handleAppRunningMessage - not stored, from ${fromIP}:${port} for ${message.data?.ip}: ${result.message}`); + return; + } if (result.stored) { fluxEventBus.publish('network:apprunning', { ip: message.data.ip, apps: message.data.apps || [{ name: message.data.name }] }); } diff --git a/ZelBack/src/services/utils/appRunningEntries.js b/ZelBack/src/services/utils/appRunningEntries.js new file mode 100644 index 0000000000..6b677f332c --- /dev/null +++ b/ZelBack/src/services/utils/appRunningEntries.js @@ -0,0 +1,25 @@ +/** + * The app entries of a version 2 apprunning message that can be stored, and the + * ones that cannot. An entry is a location row only with a string name and a + * string hash. Entries are judged one at a time, so an entry that fails costs + * its sender that entry and not the rest of the message. + * + * The announcer applies the same rule to what it sends, so a node never + * announces an entry its peers refuse. + * @param {Array<*>} apps The message's apps. + * @returns {{stored: Array, refused: Array<*>}} Storable entries and the rest. + */ +function appRunningEntries(apps) { + const stored = []; + const refused = []; + for (const app of apps) { + if (app && typeof app === 'object' && typeof app.name === 'string' && typeof app.hash === 'string') { + stored.push(app); + } else { + refused.push(app); + } + } + return { stored, refused }; +} + +module.exports = { appRunningEntries }; diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index b402df72d0..2745bd7f33 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -629,6 +629,26 @@ describe('fluxCommunication tests', () => { sinon.assert.notCalled(broadcastHashSpy); }).timeout(5000); + + // A refused message is logged where it arrived, and goes no further. + it('logs a message its store refuses, and does not relay it', async () => { + sinon.stub(messageStore, 'storeAppRunningMessage').resolves(new Error('Invalid Flux App Running message for storing')); + sinon.stub(messageStore, 'storeAppStateEvent'); + const warn = sinon.stub(log, 'warn'); + const fromIp = '127.0.0.5'; + const port = '16127'; + const message = { + data: { + type: 'fluxapprunning', version: 2, apps: 'not an array', broadcastedAt: Date.now(), ip: '203.0.113.30:16127', + }, + timestamp: Date.now(), + }; + + await fluxCommunication.handleAppRunningMessage(message, fromIp, port); + + sinon.assert.notCalled(broadcastHashSpy); + sinon.assert.calledWithMatch(warn, /handleAppRunningMessage - not stored, from 127\.0\.0\.5:16127 for 203\.0\.113\.30:16127: Invalid Flux App Running message for storing/); + }); }); describe('the installing claim handlers', () => { diff --git a/tests/unit/messageStoreAppRunningEntries.test.js b/tests/unit/messageStoreAppRunningEntries.test.js new file mode 100644 index 0000000000..171e15dff0 --- /dev/null +++ b/tests/unit/messageStoreAppRunningEntries.test.js @@ -0,0 +1,176 @@ +const { expect } = require('chai'); +const config = require('config'); +const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const messageStore = require('../../ZelBack/src/services/appMessaging/messageStore'); +const { appRunningEntries } = require('../../ZelBack/src/services/utils/appRunningEntries'); +const { RUNNING_EXPIRY_MS } = require('../../ZelBack/src/services/utils/appConstants'); +const { requireMongo } = require('./dbTestHelper'); + +// A version 2 apprunning message is judged entry by entry: one entry without a +// name and hash costs its sender that entry, never the rest of the message. +describe('apprunning entries are stored one at a time', () => { + const ip = '203.0.113.21:16127'; + const otherIp = '203.0.113.22:16127'; + const weeksAgo = new Date(Date.now() - 21 * 24 * 60 * 60 * 1000); + + describe('appRunningEntries', () => { + it('keeps an entry with a string name and a string hash', () => { + const app = { name: 'web', hash: 'h1', runningSince: weeksAgo.toISOString() }; + expect(appRunningEntries([app])).to.deep.equal({ stored: [app], refused: [] }); + }); + + it('refuses each entry without a string name and hash, and only those', () => { + const good = { name: 'web', hash: 'h1' }; + const bad = [ + { name: 'probe' }, + { name: 'probe', hash: null }, + { name: 'probe', hash: 7 }, + { hash: 'h2' }, + { name: 3, hash: 'h2' }, + null, + 'web', + undefined, + ]; + const { stored, refused } = appRunningEntries([bad[0], good, ...bad.slice(1)]); + expect(stored).to.deep.equal([good]); + expect(refused).to.deep.equal(bad); + }); + + it('answers an empty list with nothing either way', () => { + expect(appRunningEntries([])).to.deep.equal({ stored: [], refused: [] }); + }); + }); + + describe('on a database', () => { + before(requireMongo); + + let locations; + + const row = (name, rowIp, broadcastedAt, runningSince = broadcastedAt) => ({ + name, + ip: rowIp, + hash: `${name}hash`, + broadcastedAt: new Date(broadcastedAt), + expireAt: new Date(broadcastedAt + RUNNING_EXPIRY_MS), + runningSince: new Date(runningSince), + }); + const message = (apps, broadcastedAt = Date.now(), from = ip) => ({ + type: 'fluxapprunning', version: 2, apps, ip: from, broadcastedAt, osUptime: 1000, staticIp: true, + }); + const rowsAt = async (rowIp = ip) => (await locations.find({ ip: rowIp }).sort({ name: 1 }).toArray()); + + beforeEach(async () => { + const database = dbHelper.databaseConnection().db(config.database.appsglobal.database); + locations = database.collection(config.database.appsglobal.collections.appsLocations); + await locations.deleteMany({ ip: { $in: [ip, otherIp] } }); + }); + + afterEach(async () => { + await locations.deleteMany({ ip: { $in: [ip, otherIp] } }); + }); + + describe('storeAppRunningMessage', () => { + it('stores the entries with a name and hash and refuses the one without', async () => { + const result = await messageStore.storeAppRunningMessage(message([ + { name: 'web', hash: 'webhash' }, + { name: 'probe' }, + { name: 'db', hash: 'dbhash' }, + ])); + + expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect((await rowsAt()).map((r) => r.name)).to.deep.equal(['db', 'web']); + }); + + // A long-running app announced beside an app installed without a hash. + it('keeps refreshing a long-running app announced beside an entry without a hash', async () => { + const lastHour = Date.now() - 60 * 60 * 1000; + await locations.insertOne(row('hytale', ip, lastHour, weeksAgo.getTime())); + + const now = Date.now(); + await messageStore.storeAppRunningMessage(message([ + { name: 'hytale', hash: 'hytalehash', runningSince: weeksAgo.toISOString() }, + { name: 'probe', runningSince: new Date(now).toISOString() }, + ], now)); + + const [hytale, ...rest] = await rowsAt(); + expect(rest).to.deep.equal([]); + expect(hytale.name).to.equal('hytale'); + expect(hytale.broadcastedAt.getTime()).to.equal(now); + expect(hytale.expireAt.getTime()).to.equal(now + RUNNING_EXPIRY_MS); + expect(hytale.runningSince.getTime()).to.equal(weeksAgo.getTime()); + }); + + it('stores nothing and relays nothing when every entry is refused', async () => { + const result = await messageStore.storeAppRunningMessage(message([{ name: 'probe' }, null])); + + expect(result).to.deep.equal({ stored: false, rebroadcast: false }); + expect(await rowsAt()).to.deep.equal([]); + }); + + // An empty v2 message deletes the sender's rows. A message whose entries + // were all refused named apps; it must not be read as one that named none. + it('never reads a message whose every entry was refused as an empty snapshot', async () => { + const earlier = Date.now() - 10 * 60 * 1000; + await locations.insertOne(row('web', ip, earlier)); + + await messageStore.storeAppRunningMessage(message([{ name: 'probe' }])); + + const kept = await rowsAt(); + expect(kept.map((r) => r.name)).to.deep.equal(['web']); + expect(kept[0].broadcastedAt.getTime()).to.equal(earlier); + }); + + // Canary for the test above: the deletion it guards against is real. + it('still deletes the sender\'s older rows on a message that names no apps', async () => { + await locations.insertOne(row('web', ip, Date.now() - 10 * 60 * 1000)); + + const result = await messageStore.storeAppRunningMessage(message([])); + + expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(await rowsAt()).to.deep.equal([]); + }); + + it('writes nothing for another node', async () => { + await locations.insertOne(row('web', otherIp, Date.now() - 10 * 60 * 1000)); + + await messageStore.storeAppRunningMessage(message([{ name: 'probe' }, { name: 'db', hash: 'dbhash' }])); + + expect((await rowsAt(otherIp)).map((r) => r.name)).to.deep.equal(['web']); + }); + + it('still refuses a version 1 message without a hash', async () => { + const result = await messageStore.storeAppRunningMessage({ + type: 'fluxapprunning', version: 1, name: 'probe', ip, broadcastedAt: Date.now(), + }); + + expect(result).to.be.instanceOf(Error); + expect(await rowsAt()).to.deep.equal([]); + }); + }); + + describe('storeBatchAppRunningMessages', () => { + const broadcast = (data) => ({ version: 1, timestamp: data.broadcastedAt, pubKey: '04ab', signature: 'sig', data }); + + it('stores the entries with a name and hash and writes no row for the one without', async () => { + const result = await messageStore.storeBatchAppRunningMessages([ + broadcast(message([{ name: 'web', hash: 'webhash' }, { name: 'probe' }, { name: 'db', hash: 'dbhash' }])), + ]); + + expect(result.writeFailed).to.equal(false); + const rows = await rowsAt(); + expect(rows.map((r) => r.name)).to.deep.equal(['db', 'web']); + expect(rows.every((r) => typeof r.hash === 'string')).to.equal(true); + }); + + it('stores the other broadcasts of a batch when one has only refused entries', async () => { + await messageStore.storeBatchAppRunningMessages([ + broadcast(message([{ name: 'probe' }])), + broadcast(message([{ name: 'web', hash: 'webhash' }], Date.now(), otherIp)), + ]); + + expect(await rowsAt()).to.deep.equal([]); + expect((await rowsAt(otherIp)).map((r) => r.name)).to.deep.equal(['web']); + }); + }); + }); +}); From c16e73f600722e587ddee858273486dc420c65ae Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 07:29:01 +0100 Subject: [PATCH 325/369] fix(apps): a node announces only the apps a peer can store An installed app without a string name and hash is left out of the announcement and logged, so it cannot cost the node its other apps on a peer that refuses the whole message. An announcement the node's own store refuses is not broadcast. Co-Authored-By: Claude Opus 5.5 --- .../services/appMessaging/peerNotification.js | 18 ++++- tests/unit/peerNotification.test.js | 74 +++++++++++++++---- 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index de5ecc219c..f883dfed3e 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -13,6 +13,7 @@ const appReconciler = require('../appMonitoring/appReconciler'); const fluxEventBus = require('../utils/fluxEventBus'); const { nodeSigner } = require('../utils/nodeSigner'); +const { appRunningEntries } = require('../utils/appRunningEntries'); const { ANNOUNCE_INTERVAL_MS } = require('../utils/appConstants'); const globalAppsLocations = config.database.appsglobal.collections.appsLocations; @@ -196,12 +197,19 @@ async function checkAndNotifyPeersOfRunningApps() { && !globalState.testInstallingApps.has(application.name) && !globalState.unconfirmedApps.has(application.name), ); + // An installed app without a string name and hash is left out and logged: no + // node stores such an entry, and a node that has not upgraded refuses the + // whole message for it, which would cost this node every other app it holds. + const { stored: announceable, refused } = appRunningEntries(applicationsToBroadcast); + if (refused.length) { + log.warn(`checkAndNotifyPeersOfRunningApps - installed apps without a name and hash are not announced: ${refused.map((application) => application?.name).join(', ')}`); + } const apps = []; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); try { // eslint-disable-next-line no-restricted-syntax - for (const application of applicationsToBroadcast) { + for (const application of announceable) { const queryFind = { name: application.name, ip: localSocketAddr }; const projection = { _id: 0, runningSince: 1 }; // eslint-disable-next-line no-await-in-loop @@ -243,7 +251,13 @@ async function checkAndNotifyPeersOfRunningApps() { log.warn('checkAndNotifyPeersOfRunningApps - this node cannot sign as itself; its running apps are not announced'); return; } - await messageStore.storeAppRunningMessage(appRunningMessage); + // Every peer applies the same rule to the message as this store does, so a + // message this node will not store is one no peer stores either. + const ownStore = await messageStore.storeAppRunningMessage(appRunningMessage); + if (ownStore instanceof Error) { + log.error(`checkAndNotifyPeersOfRunningApps - this node refuses its own announcement, not broadcast: ${ownStore.message}`); + return; + } const signed = await fluxCommunicationMessagesSender.broadcastMessageToAll(appRunningMessage); await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed }); fluxEventBus.publish('app:running', { apps, ip: appRunningMessage.ip }); diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index b2fa209f47..f433efde0c 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -134,7 +134,7 @@ describe('peerNotification tests', () => { nodeSignerStub = sinon.stub().resolves({ pubKey: '04', sign: () => 'sig' }); installedAppsStub = sinon.stub().resolves({ status: 'success', - data: [{ name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], + data: [{ name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], }); // The REAL departing tracker, taken fresh per test rather than // reimplemented here: a fake that counts differently from the module would @@ -197,8 +197,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, ], }); // only app1's container is running @@ -215,8 +215,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, ], }); departingApps.enter('app2'); @@ -251,8 +251,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, ], }); unconfirmedApps.set('app2', Symbol('app2')); @@ -304,7 +304,7 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, { name: 'sealed', version: 8, enterprise: 'encrypted-blob', compose: [], hash: 'h2' }, ], }); @@ -344,8 +344,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'trialapp', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'trialapp', hash: 'trialapphash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, ], }); testInstallingApps.add('trialapp'); @@ -359,6 +359,52 @@ describe('peerNotification tests', () => { ).to.deep.equal(['app1']); }); + // No node stores an entry without a name and hash, and a node that has not + // upgraded refuses the whole message for one - which would cost this node + // every other app it holds. + it('leaves out an installed app without a hash and announces the rest', async () => { + installedAppsStub.resolves({ + status: 'success', + data: [ + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'probe', version: 8, compose: [{ name: 'web', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + ], + }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps.map((a) => a.name)).to.deep.equal(['app1', 'app2']); + expect(broadcastMessageToAllStub.firstCall.args[0]).to.equal(message); + expect(logStub.warn.calledWithMatch(/not announced: probe/)).to.equal(true); + }); + + it('announces nothing when no installed app has a hash', async () => { + installedAppsStub.resolves({ + status: 'success', + data: [{ name: 'probe', version: 8, compose: [{ name: 'web', containerData: '/data' }] }], + }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + expect(storeAppRunningMessageStub.called).to.equal(false); + expect(broadcastMessageToAllStub.called).to.equal(false); + }); + + // Every peer applies the store's rule, so a message this node refuses is one + // no peer keeps either. + it('broadcasts nothing its own store refuses', async () => { + storeAppRunningMessageStub.resolves(new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode')); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + expect(storeAppRunningMessageStub.calledOnce).to.equal(true); + expect(broadcastMessageToAllStub.called, 'a refused announcement is not sent').to.equal(false); + expect(storeAppStateEventStub.called).to.equal(false); + expect(logStub.error.calledWithMatch(/refuses its own announcement/)).to.equal(true); + }); + // An empty snapshot must NEVER be broadcast: on the receive side an empty v2 // message deletes every appsLocations row for the sender's IP - and the sender // stores its own message first, so it erases its own network presence. The @@ -409,9 +455,9 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'gapp', version: 4, compose: [{ name: 'gc', containerData: 'g:/data' }] }, - { name: 'rapp', version: 4, compose: [{ name: 'rc', containerData: 'r:/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'gapp', hash: 'gapphash', version: 4, compose: [{ name: 'gc', containerData: 'g:/data' }] }, + { name: 'rapp', hash: 'rapphash', version: 4, compose: [{ name: 'rc', containerData: 'r:/data' }] }, ], }); // neither gapp's nor rapp's containers are running @@ -447,7 +493,7 @@ describe('peerNotification tests', () => { if (opts.cycleMs && cycles <= (opts.slowCycles ?? 1)) clock.tick(opts.cycleMs); return { status: 'success', - data: [{ name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], + data: [{ name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], }; }); const mod = loadPeerNotification({ ...opts, installedApps }); From 550d327c4f750a3640acf31a799cb9650b437ce2 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 07:29:01 +0100 Subject: [PATCH 326/369] fix(apps): installing an app by hand is the flux team's, testing one is its owner's installapplocally installs registered apps, for the flux team only, and never from a temporary message. testappinstall tests an app not yet paid for only for that app's owner or the flux team. A specification taken from a message carries the message's hash, and from a permanent message its height, through decryption as well. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appInstaller.js | 88 ++++------- tests/unit/appInstaller.test.js | 144 ++++++++++++++++-- 2 files changed, 160 insertions(+), 72 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 6d6eb2dfc6..fefc9f6e58 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -1032,9 +1032,12 @@ async function installAppLocally(req, res) { if (!appname) { throw new Error('No Flux App specified'); } - let blockAllowance = config.fluxapps.ownerAppAllowance; - // needs to be logged in - const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); + // Placing an app on a node is not the node operator's call, for the same + // reason removing one is not: hosting an app is not owning it. The spawner + // decides placement; the owner decides everything else. An app not yet paid + // for is never placed here: it is tried out through testAppInstall, which + // tears it down again and never announces it. + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (authorized) { // registerAppLocally refuses a concurrent removal or installation but not a // redeploy or the periodic reinstall pass, and those hold the node across a @@ -1045,41 +1048,8 @@ async function installAppLocally(req, res) { if (heldBy) { throw new Error(`Another application is undergoing ${heldBy}. Installation not possible.`); } - let appSpecifications; - // anyone can deploy temporary app - // favor temporary to launch test temporary apps - const tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(appname); - if (tempMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = tempMessage.appSpecifications; - // blockAllowance is used for future validation - // eslint-disable-next-line no-unused-vars - blockAllowance = config.fluxapps.temporaryAppAllowance; - } - if (!appSpecifications) { - // Placing a registered app on a node is not the node operator's call, for - // the same reason removing one is not: hosting an app is not owning it. - // This branch resolves an app BY NAME from the marketplace, the global - // registry or a permanent message, so an operator reaching it is choosing - // which customer's app runs on hardware they control - and for a g:/r: app - // the new instance syncs that customer's data down to it. The spawner - // decides placement; the owner decides everything else. - // - // The temporary-message branch above is untouched and stays open to any - // logged-in user: that is how an app is tested before it is registered, - // it is addressed by hash rather than by name, and it expires on its own - // (temporaryAppAllowance). - const ownerAuthorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); - if (!ownerAuthorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - return; - } - } - if (!appSpecifications) { - const allApps = await availableApps(); - appSpecifications = allApps.find((app) => app.name === appname); - } + const allApps = await availableApps(); + let appSpecifications = allApps.find((app) => app.name === appname); if (!appSpecifications) { // eslint-disable-next-line no-use-before-define appSpecifications = await getApplicationGlobalSpecifications(appname); @@ -1088,8 +1058,10 @@ async function installAppLocally(req, res) { if (!appSpecifications) { const permMessage = await messageVerifier.checkAppMessageExistence(appname); if (permMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = permMessage.appSpecifications; + // The message carries the hash and height its specification does not: + // the announcement names the app by that hash, and the expiry sweep + // removes a local row without a height. + appSpecifications = { ...permMessage.appSpecifications, hash: permMessage.hash, height: permMessage.height }; } } if (!appSpecifications) { @@ -1116,7 +1088,7 @@ async function installAppLocally(req, res) { // get current height const dbopen = dbHelper.databaseConnection(); if (!appSpecifications.height && appSpecifications.height !== 0) { - // precaution for old temporary apps. Set up for custom test specifications. + // A specification without a height runs for ownerAppAllowance blocks from now. const database = dbopen.db(config.database.daemon.database); const query = { generalScannedHeight: { $gte: 0 } }; const projection = { @@ -1130,7 +1102,7 @@ async function installAppLocally(req, res) { throw new Error('Scanning not initiated'); } const explorerHeight = serviceHelper.ensureNumber(result.generalScannedHeight); - appSpecifications.height = explorerHeight - config.fluxapps.blocksLasting + blockAllowance; // allow running for this amount of blocks + appSpecifications.height = explorerHeight - config.fluxapps.blocksLasting + config.fluxapps.ownerAppAllowance; } const appsDatabase = dbopen.db(config.database.appslocal.database); @@ -1225,7 +1197,6 @@ async function testAppInstall(req, res) { } log.info(`testAppInstall: ${appname}`); - let blockAllowance = config.fluxapps.ownerAppAllowance; // needs to be logged in const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); @@ -1238,15 +1209,20 @@ async function testAppInstall(req, res) { } let appSpecifications; - // anyone can deploy temporary app - // favor temporary to launch test temporary apps + // An app not yet paid for is tested by its owner, addressed by its message + // hash. The flux team may test any. const tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(appname); if (tempMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = tempMessage.appSpecifications; - // blockAllowance is used for future validation - // eslint-disable-next-line no-unused-vars - blockAllowance = config.fluxapps.temporaryAppAllowance; + const caller = serviceHelper.ensureObject(authOf(req))?.zelid; + const mayTest = caller === tempMessage.appSpecifications?.owner + || await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (!mayTest) { + const errMessage = messageHelper.errUnauthorizedMessage(); + res.json(errMessage); + return; + } + // The hash is the message's, not the specification's. + appSpecifications = { ...tempMessage.appSpecifications, hash: tempMessage.hash }; } if (!appSpecifications) { @@ -1257,11 +1233,6 @@ async function testAppInstall(req, res) { // which customer's app runs on hardware they control - and for a g:/r: app // the new instance syncs that customer's data down to it. The spawner // decides placement; the owner decides everything else. - // - // The temporary-message branch above is untouched and stays open to any - // logged-in user: that is how an app is tested before it is registered, - // it is addressed by hash rather than by name, and it expires on its own - // (temporaryAppAllowance). const ownerAuthorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (!ownerAuthorized) { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -1283,8 +1254,7 @@ async function testAppInstall(req, res) { if (!appSpecifications) { const permMessage = await messageVerifier.checkAppMessageExistence(appname); if (permMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = permMessage.appSpecifications; + appSpecifications = { ...permMessage.appSpecifications, hash: permMessage.hash, height: permMessage.height }; } } @@ -1305,11 +1275,15 @@ async function testAppInstall(req, res) { } const daemonHeight = syncStatus.data.height; + // the formatter strips spec metadata; the hash and height must survive it + const { height, hash } = appSpecifications; appSpecifications = await checkAndDecryptAppSpecs(appSpecifications, { daemonHeight, owner: appSpecifications.owner, }); appSpecifications = specificationFormatter(appSpecifications); + appSpecifications.height = height; + appSpecifications.hash = hash; } // Test installation - similar to regular install but with test flag diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index e3d15e09ee..5b293625ba 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -148,6 +148,7 @@ describe('appInstaller tests', () => { '../serviceHelper': { ensureString: sinon.stub().returnsArg(0), ensureNumber: sinon.stub().returnsArg(0), + ensureObject: (value) => (typeof value === 'string' ? JSON.parse(value) : value), delay: sinon.stub().resolves(), }, '../generalService': { @@ -386,7 +387,7 @@ describe('appInstaller tests', () => { await appInstaller.installAppLocally(req, res); expect(res.json.calledOnce).to.be.true; - expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.USER, authOf(req))).to.be.true; + expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.FLUX_TEAM, authOf(req))).to.be.true; }); it('should handle missing appname parameter', async () => { @@ -437,9 +438,9 @@ describe('appInstaller tests', () => { }); }); - // Placing a registered app on a node is not the operator's call, for the same - // reason removing one is not. The temporary-message route is deliberately NOT - // closed with it - that is how an app is tested before it is registered. + // Placing an app on a node is not the operator's call, for the same reason + // removing one is not, and an app not yet paid for is never placed: it is + // tried out through the test-install route, which tears it down again. describe('local install is not the node operator\'s to make', () => { const nameInstall = () => ({ params: { appname: 'someCustomerApp' }, query: {} }); @@ -477,27 +478,137 @@ describe('appInstaller tests', () => { expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); }); - // The one that must keep working: an app under test is addressed by HASH and - // carries a temporary message, so it never reaches the by-name gate at all. - it('still lets any logged-in user install an app under test by its temporary message', async () => { - const req = { params: { appname: 'a1b2c3hash' }, query: {} }; + it('refuses a logged-in user installing an app by its temporary message', async () => { + const req = { params: { appname: 'a1b2c3hash' }, query: {}, headers: { zelidauth: JSON.stringify({ zelid: 'someone' }) } }; const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); - verificationHelperStub.verifyPrivilege.withArgs(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)).resolves(false); verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); messageVerifierStub.checkAppTemporaryMessageExistence.resolves({ + hash: 'a1b2c3hash', appSpecifications: { name: 'apptest', version: 3, owner: 'someone' }, }); messageHelperStub.errUnauthorizedMessage.returns({ status: 'error', data: { message: 'Unauthorized' } }); await appInstaller.installAppLocally(req, res); - // It got past the gate: no Unauthorized was returned, and the by-name - // privilege was never consulted because a temporary message was found. - const unauthorized = res.json.getCalls().some((c) => c.args[0]?.data?.message === 'Unauthorized'); - expect(unauthorized, 'a temporary app must not be refused').to.equal(false); - expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.FLUX_TEAM, authOf(req)), 'the by-name gate must not be reached').to.be.false; + expect(res.json.calledOnce).to.be.true; + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); + expect(dbHelperStub.insertOneToDatabase.called, 'nothing is installed').to.be.false; + }); + + it('does not install a temporary message for the flux team either', async () => { + const req = { params: { appname: 'a1b2c3hash' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + messageVerifierStub.checkAppTemporaryMessageExistence.resolves({ + hash: 'a1b2c3hash', + appSpecifications: { name: 'apptest', version: 3, owner: 'someone' }, + }); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(res.json.calledOnce).to.be.true; + expect(res.json.firstCall.args[0].data.message).to.equal('Application Specifications of a1b2c3hash not found'); + expect(hwRequirementsStub.checkAppHWRequirements.called, 'nothing is installed').to.be.false; + }); + + it('installs a permanent message with the hash and height the message carries', async () => { + const req = { params: { appname: 'permhash' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + messageVerifierStub.checkAppMessageExistence.withArgs('permhash').resolves({ + hash: 'permhash', + height: 2000000, + appSpecifications: { name: 'permapp', version: 3, owner: 'someone' }, + }); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + // stop at the requirement check: the specification it is given is what gets installed + hwRequirementsStub.checkAppHWRequirements.rejects(new Error('stop here')); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(hwRequirementsStub.checkAppHWRequirements.calledOnce).to.be.true; + const installed = hwRequirementsStub.checkAppHWRequirements.firstCall.args[0]; + expect(installed).to.include({ name: 'permapp', hash: 'permhash', height: 2000000 }); + }); + }); + + // An app not yet paid for is tried out by its owner. Anyone else is refused + // before a container is made, and the flux team may test any. + describe('testAppInstall of an app not yet paid for', () => { + const owner = '1OwnerOfTheTemporaryApp'; + const tempMessage = { + hash: 'temphash', + appSpecifications: { + name: 'trialapp', version: 4, owner, compose: [{ name: 'web', repotag: 'org/web:1', cpu: 0.5, ram: 500, hdd: 5 }], + }, + }; + const requestAs = (zelid) => ({ params: { appname: 'temphash' }, query: {}, headers: { zelidauth: JSON.stringify({ zelid, loginPhrase: 'p', signature: 's' }) } }); + const response = () => ({ json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }); + // reached only once the caller has been let through + const pastTheGate = () => baseStubs['../appSystem/systemIntegration'].systemArchitecture.called; + + beforeEach(() => { + messageVerifierStub.checkAppTemporaryMessageExistence.withArgs('temphash').resolves(tempMessage); + messageHelperStub.errUnauthorizedMessage.returns({ status: 'error', data: { message: 'Unauthorized' } }); + // stop at the architecture check: past it the install itself runs + baseStubs['../appSystem/systemIntegration'].systemArchitecture.rejects(new Error('stop here')); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + }); + + it('lets the owner test it', async () => { + const req = requestAs(owner); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(true); + expect(res.json.calledWithMatch({ data: { message: 'Unauthorized' } })).to.equal(false); + }); + + it('refuses another logged-in user', async () => { + const req = requestAs('1SomeoneElse'); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + expect(res.json.calledOnce).to.equal(true); + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); + }); + + it('lets the flux team test it', async () => { + const req = requestAs('1FluxTeam'); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(true); + }); + + it('refuses a caller with no session at all', async () => { + const req = { params: { appname: 'temphash' }, query: {} }; + const res = response(); + verificationHelperStub.verifyPrivilege.resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); }); }); @@ -613,6 +724,7 @@ describe('appInstaller tests', () => { // Mock message verifier to return enterprise app with empty compose messageVerifierStub.checkAppTemporaryMessageExistence.resolves({ + hash: 'enterprisehash', appSpecifications: enterpriseAppSpec, }); @@ -629,8 +741,10 @@ describe('appInstaller tests', () => { } // Verify that decryption was called for enterprise app - expect(enterpriseHelperStub.checkAndDecryptAppSpecs.calledWith(enterpriseAppSpec)).to.be.true; + expect(enterpriseHelperStub.checkAndDecryptAppSpecs.calledWith({ ...enterpriseAppSpec, hash: 'enterprisehash' })).to.be.true; expect(appSpecHelpersStub.specificationFormatter.calledWith(decryptedAppSpec)).to.be.true; + // the formatter's output keeps the hash the message carried + expect(decryptedAppSpec.hash).to.equal('enterprisehash'); expect(logStub.info.calledWith('testAppInstall: enterpriseapp')).to.be.true; }); From 127eed93852f63fe78d056df74ba813797f09ca3 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 07:42:55 +0100 Subject: [PATCH 327/369] fix(apps): a sync batch logs its refused apprunning entries in one line A sync response is stored a slice at a time and a slice holds thousands of broadcasts, so the entries one call refuses are summarised in a single warning rather than one per broadcast. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMessaging/messageStore.js | 24 ++++++++++++------- .../messageStoreAppRunningEntries.test.js | 17 +++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index c34742a067..e0c9af24e9 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -290,15 +290,17 @@ function runningSinceOf(isNewer, runningSince, moved) { } /** - * Log the entries of one apprunning message that were not stored. + * Log, in one line, the apprunning entries a store refused. * @param {string} caller The function that refused them. - * @param {object} message The message's data. - * @param {Array<*>} refused The refused entries. + * @param {Array<{ip: string, broadcastedAt: number, refused: Array<*>}>} refusals One per message with refused entries. */ -function logRefusedEntries(caller, message, refused) { - if (!refused.length) return; - const shown = refused.slice(0, 5).map((app) => JSON.stringify(app)?.slice(0, 200)).join(', '); - log.warn(`${caller} - ${refused.length} app entries from ${message.ip} at ${message.broadcastedAt} have no string name and hash and are not stored: ${shown}`); +function logRefusedEntries(caller, refusals) { + if (!refusals.length) return; + const entries = refusals.reduce((total, { refused }) => total + refused.length, 0); + const shown = refusals.slice(0, 3).map(({ ip, broadcastedAt, refused }) => ( + `${ip} at ${broadcastedAt}: ${refused.slice(0, 3).map((app) => JSON.stringify(app)?.slice(0, 200)).join(', ')}` + )).join('; '); + log.warn(`${caller} - ${entries} app entries in ${refusals.length} messages have no string name and hash and are not stored. ${shown}`); } /** @@ -354,7 +356,7 @@ async function storeAppRunningMessage(message) { return new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode'); } const { stored, refused } = appRunningEntries(message.apps); - logRefusedEntries('storeAppRunningMessage', message, refused); + if (refused.length) logRefusedEntries('storeAppRunningMessage', [{ ip: message.ip, broadcastedAt: message.broadcastedAt, refused }]); if (message.apps.length && !stored.length) { return { stored: false, rebroadcast: false }; } @@ -807,6 +809,9 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { verifiedBroadcasts.map(({ data }) => ({ ip: data.ip, at: data.broadcastedAt })), ); + // one log line per call: a sync response is stored a slice at a time, and a + // slice holds thousands of broadcasts + const refusals = []; for (const [index, broadcast] of verifiedBroadcasts.entries()) { const { data } = broadcast; const validTill = data.broadcastedAt + RUNNING_EXPIRY_MS; @@ -814,7 +819,7 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { const ip = addresses[index]; const { stored: apps, refused } = appRunningEntries(data.version === 2 ? (data.apps || []) : [{ name: data.name, hash: data.hash }]); - logRefusedEntries('storeBatchAppRunningMessages', data, refused); + if (refused.length) refusals.push({ ip: data.ip, broadcastedAt: data.broadcastedAt, refused }); const incomingDate = new Date(data.broadcastedAt); const incomingExpiry = new Date(validTill); const isNewer = { $gt: [incomingDate, { $ifNull: ['$broadcastedAt', new Date(0)] }] }; @@ -848,6 +853,7 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { } await flushLocationOps(); + logRefusedEntries('storeBatchAppRunningMessages', refusals); // Upserts only. Removing the rows a node no longer reports belongs to // pruneAppRunningLocations, which is driven from the newest broadcast per node diff --git a/tests/unit/messageStoreAppRunningEntries.test.js b/tests/unit/messageStoreAppRunningEntries.test.js index 171e15dff0..49da92a784 100644 --- a/tests/unit/messageStoreAppRunningEntries.test.js +++ b/tests/unit/messageStoreAppRunningEntries.test.js @@ -1,7 +1,9 @@ const { expect } = require('chai'); +const sinon = require('sinon'); const config = require('config'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const messageStore = require('../../ZelBack/src/services/appMessaging/messageStore'); +const log = require('../../ZelBack/src/lib/log'); const { appRunningEntries } = require('../../ZelBack/src/services/utils/appRunningEntries'); const { RUNNING_EXPIRY_MS } = require('../../ZelBack/src/services/utils/appConstants'); const { requireMongo } = require('./dbTestHelper'); @@ -162,6 +164,21 @@ describe('apprunning entries are stored one at a time', () => { expect(rows.every((r) => typeof r.hash === 'string')).to.equal(true); }); + it('logs the refusals of a whole batch in one line', async () => { + const warn = sinon.stub(log, 'warn'); + try { + await messageStore.storeBatchAppRunningMessages([ + broadcast(message([{ name: 'probe' }, { name: 'web', hash: 'webhash' }])), + broadcast(message([{ name: 'probe2' }, { name: 'probe3' }], Date.now(), otherIp)), + ]); + } finally { + warn.restore(); + } + + expect(warn.callCount).to.equal(1); + expect(warn.firstCall.args[0]).to.match(/^storeBatchAppRunningMessages - 3 app entries in 2 messages have no string name and hash and are not stored\./); + }); + it('stores the other broadcasts of a batch when one has only refused entries', async () => { await messageStore.storeBatchAppRunningMessages([ broadcast(message([{ name: 'probe' }])), From 0bbbad41456d3107af891ebc253fe7ec68dea5b1 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 07:51:15 +0100 Subject: [PATCH 328/369] fix(apps): a node keeps its own record of when it took each app zelappsplacements (localzelapps) holds one record per installed app: when this node took it. The announcer reads runningSince from it, never from the network's copy of this node's location rows, which lapse in normal running while the app goes on running. - Written when an app is installed, and never moved: a redeploy's reinstall finds the record kept and leaves it. - removeAppLocally deletes it with the app. A removal that is part of a redeploy passes { redeploying: true } and keeps it: the hard redeploy and spec-update rebuild, and the component-structure-change redeploy. Either deletes it when the app does not come back. - At boot, before the first announcement, the records are made to match the installed apps. An app without one is dated at the runningSince of this node's own location row, so it keeps the place among its app's instances the network already gives it; without a row, now. A record for an app not installed here is removed. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 1 + .../appLifecycle/advancedWorkflows.js | 6 +- .../src/services/appLifecycle/appInstaller.js | 4 + .../src/services/appLifecycle/appRebuild.js | 9 +- .../appLifecycle/appStartupManager.js | 19 +++ .../services/appLifecycle/appUninstaller.js | 8 +- .../services/appManagement/appPlacements.js | 123 ++++++++++++++ .../services/appMessaging/peerNotification.js | 28 ++- ZelBack/src/services/serviceManager.js | 3 + tests/unit/appInstaller.test.js | 44 +++++ tests/unit/appPlacements.test.js | 160 ++++++++++++++++++ tests/unit/appRebuild.test.js | 47 +++++ tests/unit/appStartupManager.test.js | 27 ++- tests/unit/appUninstaller.test.js | 62 +++++++ tests/unit/globalconfig/default.js | 1 + tests/unit/peerNotification.test.js | 49 ++++++ 16 files changed, 579 insertions(+), 12 deletions(-) create mode 100644 ZelBack/src/services/appManagement/appPlacements.js create mode 100644 tests/unit/appPlacements.test.js diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index df3e05bc7e..d4a1e63590 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -102,6 +102,7 @@ module.exports = { collections: { appsInformation: 'zelappsinformation', appsRuntimeState: 'zelappsruntimestate', // node-local per-component controller state: desiredState, restartHistory (crash backoff), last exit + appsPlacements: 'zelappsplacements', // when this node took each app it holds, kept through redeploys }, }, appsglobal: { diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 9854e662a5..5363f4a98a 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -21,6 +21,7 @@ const fluxEventBus = require('../utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../utils/addressMoveEvents'); const { InstallOutcome } = require('../utils/installOutcome'); const appRebuild = require('./appRebuild'); +const appPlacements = require('../appManagement/appPlacements'); const generalService = require('../generalService'); const placementFeasibility = require('../appPlacement/placementFeasibility'); // eslint-disable-next-line no-unused-vars @@ -4499,7 +4500,7 @@ async function reinstallOldApplications() { log.warn(`REMOVAL REASON: Component structure change (v8+) - ${appSpecifications.name} component count/names changed`); // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appSpecifications.name, null, false, false); + await appUninstaller.removeAppLocally(appSpecifications.name, null, false, false, false, { redeploying: true }); const appRedeployResponse = messageHelper.createSuccessMessage('Application removed. Awaiting installation...'); log.info(appRedeployResponse); @@ -4518,6 +4519,9 @@ async function reinstallOldApplications() { // so this node has simply lost the instance until it is placed // here again. log.error(`Application ${appSpecifications.name} was removed for a component structure change and NOT reinstalled (${outcome}). This node no longer holds it and cannot recover it on its own.`); + // eslint-disable-next-line no-await-in-loop + await appPlacements.removePlacement(appSpecifications.name) + .catch((error) => log.error(`the placement of ${appSpecifications.name} was not removed: ${error.message}`)); } // eslint-disable-next-line no-continue diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index fefc9f6e58..8f9bd5b64f 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -35,6 +35,7 @@ const { ensureStignoreCovers } = require('../appSystem/syncthingIgnorePolicy'); const { findCommonArchitectures } = require('../utils/appUtilities'); const log = require('../../lib/log'); const { localAppsInformation, scannedHeightCollection } = require('../utils/appConstants'); +const appPlacements = require('../appManagement/appPlacements'); const messageVerifier = require('../appMessaging/messageVerifier'); const { availableApps, getApplicationGlobalSpecifications } = require('../appDatabase/registryManager'); const hwRequirements = require('../appRequirements/hwRequirements'); @@ -547,6 +548,9 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f // until its monitor has measured what it holds. folderStates.installed(appSpecifications); claimed = true; + // A redeploy reinstalls an app whose record this node kept; only a new + // placement writes one. + if (!test) await appPlacements.recordPlacement(appSpecifications.name); const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; const cpuTier = `cpu${tier}`; diff --git a/ZelBack/src/services/appLifecycle/appRebuild.js b/ZelBack/src/services/appLifecycle/appRebuild.js index bdaae867e4..22b971c8b3 100644 --- a/ZelBack/src/services/appLifecycle/appRebuild.js +++ b/ZelBack/src/services/appLifecycle/appRebuild.js @@ -7,6 +7,7 @@ const log = require('../../lib/log'); const fluxEventBus = require('../utils/fluxEventBus'); const appUninstaller = require('./appUninstaller'); const appInstaller = require('./appInstaller'); +const appPlacements = require('../appManagement/appPlacements'); const { RemovalOutcome } = require('../utils/removalOutcome'); const { InstallOutcome } = require('../utils/installOutcome'); @@ -24,7 +25,7 @@ const { InstallOutcome } = require('../utils/installOutcome'); * node, FAILED when the removal did not finish. */ async function rebuildApp(appSpecs, res) { - const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); + const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false, false, { redeploying: true }); if (removal !== RemovalOutcome.REMOVED && removal !== RemovalOutcome.NOT_INSTALLED) { log.warn(`appRebuild - ${appSpecs.name} was not removed (${removal}), so it is not reinstalled`); const outcome = removal === RemovalOutcome.BUSY ? InstallOutcome.BUSY : InstallOutcome.FAILED; @@ -42,6 +43,12 @@ async function rebuildApp(appSpecs, res) { outcome = InstallOutcome.FAILED; } + // The removal kept this node's record of when it took the app, for the + // reinstall; an app that is not back here has left, and the record with it. + if (outcome !== InstallOutcome.INSTALLED) { + await appPlacements.removePlacement(appSpecs.name) + .catch((error) => log.error(`appRebuild - the placement of ${appSpecs.name} was not removed: ${error.message}`)); + } if (outcome !== InstallOutcome.INSTALLED && removal === RemovalOutcome.REMOVED) { log.warn(`appRebuild - ${appSpecs.name} was removed and not reinstalled (${outcome}); telling the network`); await appUninstaller.announceRemoval(appSpecs.name) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 6eaa15fb06..7ecdb55b27 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -18,6 +18,7 @@ const appUninstaller = require('./appUninstaller'); const appNetworkLinker = require('./appNetworkLinker'); const appSlice = require('./appSlice'); const nodeStatusMonitor = require('../appMonitoring/nodeStatusMonitor'); +const appPlacements = require('../appManagement/appPlacements'); const { TIMED_OUT } = require('../utils/withinDeadline'); const portManager = require('../appNetwork/portManager'); const globalState = require('../utils/globalState'); @@ -395,6 +396,23 @@ async function removeAllApps(reason) { } } +/** + * Make this node's placement records match the apps installed here, before its + * first announcement reads them. A failure is logged and boot goes on: an app + * without a record is announced from its location row, as it is without one at + * any time. + * @returns {Promise} + */ +async function reconcilePlacementsOnBoot() { + try { + const installed = await getInstalledAppsFromDb(); + const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); + await appPlacements.reconcilePlacements(installed.map((app) => app.name), localSocketAddr); + } catch (error) { + log.error(`appStartupManager - placement records not reconciled: ${error.message}`); + } +} + async function manageAppsOnBoot(bootContext) { try { // ahead of every container start, so no slice in use can be empty here @@ -457,6 +475,7 @@ async function manageAppsOnBoot(bootContext) { } log.info('appStartupManager - Daemon, DB, and node confirmed, reconciling apps'); + await reconcilePlacementsOnBoot(); await openInstalledAppPorts(); await reconcileAppsOnBoot(); } finally { diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 8e3305900f..ee59c37900 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -22,6 +22,7 @@ const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const { stopAppMonitoring } = require('../appManagement/appInspector'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); +const appPlacements = require('../appManagement/appPlacements'); const volumeService = require('../utils/volumeService'); const fluxEventBus = require('../utils/fluxEventBus'); const appSlice = require('./appSlice'); @@ -912,9 +913,13 @@ async function softUninstallApplication(appName, appId, appSpecifications, res, * @param {boolean} force - Force removal * @param {boolean} endResponse - Whether to end response * @param {boolean} sendMessage - Whether to send message to network + * @param {object} [options] + * @param {boolean} [options.redeploying] The app is removed to be installed here again, + * so this node keeps its record of when it took the app. Without it the app is + * leaving this node and the record goes with it. * @returns {Promise} One of RemovalOutcome. BUSY carries no claim about the app. */ -async function removeAppLocally(app, res, force = false, endResponse = true, sendMessage = false) { +async function removeAppLocally(app, res, force = false, endResponse = true, sendMessage = false, { redeploying = false } = {}) { // Names what this call marked as departing, and nothing else: the guards below // return through the same finally without having marked anything, and a refused // duplicate must not unmark the removal that is actually running. @@ -1148,6 +1153,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); + if (!redeploying) await appPlacements.removePlacement(appName); // The app is gone for good: nothing reconciles an app with no row, so its // removal records have no reader left. Only this full-uninstall path clears // them - softRemoveAppLocally deletes the row too, but as one step of a diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js new file mode 100644 index 0000000000..833e3cc1ca --- /dev/null +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -0,0 +1,123 @@ +const config = require('config'); +const log = require('../../lib/log'); +const dbHelper = require('../dbHelper'); + +// When this node took each app it holds: one record per app, keyed by name. +// +// Written when an app is installed here and kept through every redeploy and spec +// update, which remove and reinstall the app without it leaving the node. Deleted +// when the app leaves. The announcer reads runningSince from here and nowhere +// else, because the network's copy of this node's own location rows lapses in +// normal running - expiry, eviction, sigterm - while the app goes on running. +// +// Kept apart from the app's specification in appsInformation: a spec update +// deletes and re-inserts that, and this record must outlive it. + +const appsLocalDatabase = config.database.appslocal.database; +const { appsPlacements } = config.database.appslocal.collections; +const { appsLocations } = config.database.appsglobal.collections; + +function localDatabase() { + return dbHelper.databaseConnection().db(appsLocalDatabase); +} + +/** + * One record per app name. + * @returns {Promise} + */ +async function prepareCollection() { + await dbHelper.ensureIndexes(localDatabase().collection(appsPlacements), [ + { key: { name: 1 }, unique: true, name: 'name_unique', recover: dbHelper.dedupeByKey }, + ]); +} + +/** + * Record that this node took an app at `placedAt`, unless a record for it exists. + * @param {string} name App name. + * @param {Date} [placedAt] When this node took it. + * @returns {Promise} + */ +async function recordPlacement(name, placedAt = new Date()) { + // Two writers racing the first insert leave one record: the server retries an + // upsert that collides on a unique index matched by equality. + await dbHelper.updateOneInDatabase( + localDatabase(), + appsPlacements, + { name }, + { $setOnInsert: { name, placedAt } }, + { upsert: true }, + ); +} + +/** + * When this node took each named app, for the apps it has a record of. + * @param {Array} names App names. + * @returns {Promise>} placedAt by name. + */ +async function placementsOf(names) { + const records = await dbHelper.findInDatabase( + localDatabase(), + appsPlacements, + { name: { $in: names } }, + { projection: { _id: 0, name: 1, placedAt: 1 } }, + ); + return new Map(records.map(({ name, placedAt }) => [name, placedAt])); +} + +/** + * Forget an app this node no longer holds. + * @param {string} name App name. + * @returns {Promise} + */ +async function removePlacement(name) { + await dbHelper.removeDocumentsFromCollection(localDatabase(), appsPlacements, { name }); +} + +/** + * Make the records match the apps installed here, once, before this node first + * announces them. + * + * An installed app with no record gets one dated at the runningSince this node's + * own location row holds, so it keeps the place among its app's instances the + * network already gives it, and dated now when there is no row. A record for an + * app not installed here is removed, so a later install starts a new one. + * @param {Array} installedNames Names of the apps installed here. + * @param {string} localSocketAddr This node's address. + * @returns {Promise<{created: number, removed: number}>} What changed. + */ +async function reconcilePlacements(installedNames, localSocketAddr) { + const database = localDatabase(); + const recorded = await placementsOf(installedNames); + const missing = installedNames.filter((name) => !recorded.has(name)); + + const ownRows = missing.length && localSocketAddr + ? await dbHelper.findInDatabase( + dbHelper.databaseConnection().db(config.database.appsglobal.database), + appsLocations, + { ip: localSocketAddr, name: { $in: missing } }, + { projection: { _id: 0, name: 1, runningSince: 1 } }, + ) + : []; + const rowSince = new Map(ownRows.filter((row) => row.runningSince).map((row) => [row.name, new Date(row.runningSince)])); + const now = new Date(); + // eslint-disable-next-line no-restricted-syntax + for (const name of missing) { + // eslint-disable-next-line no-await-in-loop + await recordPlacement(name, rowSince.get(name) ?? now); + } + + const removal = await dbHelper.removeDocumentsFromCollection(database, appsPlacements, { name: { $nin: installedNames } }); + const removed = removal?.deletedCount ?? 0; + if (missing.length || removed) { + log.info(`appPlacements - recorded ${missing.length} placement(s) (${rowSince.size} from this node's location rows), removed ${removed} for apps not installed here`); + } + return { created: missing.length, removed }; +} + +module.exports = { + prepareCollection, + recordPlacement, + placementsOf, + removePlacement, + reconcilePlacements, +}; diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index f883dfed3e..fe1432de8e 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -14,6 +14,7 @@ const appReconciler = require('../appMonitoring/appReconciler'); const fluxEventBus = require('../utils/fluxEventBus'); const { nodeSigner } = require('../utils/nodeSigner'); const { appRunningEntries } = require('../utils/appRunningEntries'); +const appPlacements = require('../appManagement/appPlacements'); const { ANNOUNCE_INTERVAL_MS } = require('../utils/appConstants'); const globalAppsLocations = config.database.appsglobal.collections.appsLocations; @@ -208,22 +209,33 @@ async function checkAndNotifyPeersOfRunningApps() { const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); try { + // When this node took each app comes from its own record of that, never from + // the network's copy of its location rows: those lapse in normal running + // while the app goes on running here. An app with no record - the boot step + // writes one for every installed app - is announced from its row, and from + // now without one. + const placements = await appPlacements.placementsOf(announceable.map((application) => application.name)); + const unrecorded = []; // eslint-disable-next-line no-restricted-syntax for (const application of announceable) { - const queryFind = { name: application.name, ip: localSocketAddr }; - const projection = { _id: 0, runningSince: 1 }; - // eslint-disable-next-line no-await-in-loop - const result = await dbHelper.findOneInDatabase(database, globalAppsLocations, queryFind, projection); - let runningOnMyNodeSince = new Date().toISOString(); - if (result && result.runningSince) { - runningOnMyNodeSince = result.runningSince; + let runningOnMyNodeSince = placements.get(application.name); + if (!runningOnMyNodeSince) { + unrecorded.push(application.name); + const queryFind = { name: application.name, ip: localSocketAddr }; + const projection = { _id: 0, runningSince: 1 }; + // eslint-disable-next-line no-await-in-loop + const result = await dbHelper.findOneInDatabase(database, globalAppsLocations, queryFind, projection); + runningOnMyNodeSince = result?.runningSince ?? new Date(); } apps.push({ name: application.name, hash: application.hash, - runningSince: runningOnMyNodeSince, + runningSince: new Date(runningOnMyNodeSince).toISOString(), }); } + if (unrecorded.length) { + log.warn(`checkAndNotifyPeersOfRunningApps - no placement record for ${unrecorded.join(', ')}; announced from the location table`); + } // An empty snapshot is NEVER broadcast: the receive side treats an empty // v2 message as "delete every appsLocations row for this IP" - and we // store our own message first, so it would erase our own presence. Every diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 55edc4d979..9dc4ae072c 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -73,6 +73,7 @@ const uplinkService = require('./uplinkService'); const nodeConfirmationService = require('./nodeConfirmationService'); const appTamperingDetectionService = require('./appTamperingDetectionService'); const appsRuntimeState = require('./appManagement/appsRuntimeState'); +const appPlacements = require('./appManagement/appPlacements'); const imageUpdateService = require('./imageUpdateService'); const { version: fluxVersion } = require('../../../package.json'); // const throughputLogger = require('./utils/throughputLogger'); @@ -167,6 +168,8 @@ async function prepareDatabaseSchema() { // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, // then enforce one doc per component identifier await appsRuntimeState.prepareCollection(); + // appPlacements (localzelapps): one record per app + await appPlacements.prepareCollection(); // The explorer's collections: their indexes are the explorer's to declare. await explorerService.prepareCollections(); log.info('Local database prepared'); diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 5b293625ba..74ec310ef2 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -33,6 +33,13 @@ const makeDockerServiceStub = (overrides = {}) => ({ }); describe('appInstaller tests', () => { + // The placement record is its own module with its own tests; here it is the + // seam an install writes through. Fresh per test. + let appPlacementsStub; + beforeEach(() => { + appPlacementsStub = { recordPlacement: sinon.stub().resolves() }; + }); + let appInstaller; // The same stub map the default instance is built from, so a test that needs one // more seam spreads it rather than restating eighty lines of it. @@ -141,6 +148,7 @@ describe('appInstaller tests', () => { // Proxy require baseStubs = { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -790,6 +798,7 @@ describe('appInstaller tests', () => { }; const appInstallerForArchTest = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -904,6 +913,7 @@ describe('appInstaller tests', () => { }; const appInstallerForArchTest = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1138,6 +1148,7 @@ describe('appInstaller tests', () => { it('should return false if node tier does not return anything', async () => { const appInstallerWithNodeTier = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1252,6 +1263,7 @@ describe('appInstaller tests', () => { // present for that reason, and appQueryService - the first call PAST the guard - rejects, // so a guard that stops matching lands on FAILED rather than running on into the install. const installerReachingTheAlreadyInstalledGuard = () => proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../messageHelper': messageHelperStub, '../dbHelper': { @@ -1336,6 +1348,7 @@ describe('appInstaller tests', () => { }; return proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1559,6 +1572,34 @@ describe('appInstaller tests', () => { ).to.be.false; }); + // When this node took the app is written with the app's row; a redeploy's + // reinstall finds the record it kept and leaves it alone. + describe('the placement record', () => { + it('is written with the app\'s row', async () => { + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves()); + + await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, false, true); + + sinon.assert.calledOnceWithExactly(appPlacementsStub.recordPlacement, 'testapp'); + }); + + it('is not written for a test install', async () => { + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves()); + + await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, true); + + sinon.assert.notCalled(appPlacementsStub.recordPlacement); + }); + + it('is not written when the install gave up before the app\'s row', async () => { + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves(), { failBeforeRow: true }); + + await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, false, true); + + sinon.assert.notCalled(appPlacementsStub.recordPlacement); + }); + }); + it('runs the post-install broadcast only AFTER releasing the install lock', async () => { // The announcement runs for as long as a broadcast cycle takes, and every // other install, removal and redeploy on this node refuses while the install @@ -1583,6 +1624,7 @@ describe('appInstaller tests', () => { }; const appInstallerSuccess = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1770,6 +1812,7 @@ describe('appInstaller tests', () => { upnpStub = { isUPNP: sinon.stub().returns(true), mapUpnpPort: sinon.stub().resolves(MAPPED) }; logStub = { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }; appInstallerPorts = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../fluxNetworkHelper': firewallStub, '../upnpService': upnpStub, '../../lib/log': logStub, @@ -1839,6 +1882,7 @@ describe('appInstaller tests', () => { dockerServiceStub = makeDockerServiceStub({ getFreeFluxAppNetworkOctet: sinon.stub().resolves(7) }); egressRulesStub = sinon.stub().resolves(true); appInstallerNet = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../serviceHelper': { ensureString: sinon.stub().returnsArg(0) }, '../dockerService': dockerServiceStub, '../fluxNetworkHelper': { applyContainerEgressRules: egressRulesStub }, diff --git a/tests/unit/appPlacements.test.js b/tests/unit/appPlacements.test.js new file mode 100644 index 0000000000..ba86df70df --- /dev/null +++ b/tests/unit/appPlacements.test.js @@ -0,0 +1,160 @@ +const { expect } = require('chai'); +const config = require('config'); +const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const appPlacements = require('../../ZelBack/src/services/appManagement/appPlacements'); +const { requireMongo } = require('./dbTestHelper'); + +describe('appPlacements', () => { + before(requireMongo); + + const me = '203.0.113.50:16127'; + const other = '203.0.113.51:16127'; + const june = new Date('2026-06-01T00:00:00.000Z'); + const july = new Date('2026-07-01T00:00:00.000Z'); + let placements; + let locations; + + const recorded = async () => (await placements.find({}, { projection: { _id: 0 } }).sort({ name: 1 }).toArray()); + + beforeEach(async () => { + placements = dbHelper.databaseConnection().db(config.database.appslocal.database) + .collection(config.database.appslocal.collections.appsPlacements); + locations = dbHelper.databaseConnection().db(config.database.appsglobal.database) + .collection(config.database.appsglobal.collections.appsLocations); + await placements.deleteMany({}); + await locations.deleteMany({ ip: { $in: [me, other] } }); + await appPlacements.prepareCollection(); + }); + + afterEach(async () => { + await placements.deleteMany({}); + await locations.deleteMany({ ip: { $in: [me, other] } }); + }); + + it('holds one record per app name', async () => { + const indexes = await placements.indexes(); + expect(indexes.find((index) => index.name === 'name_unique')).to.include({ unique: true }); + }); + + describe('recordPlacement', () => { + it('records when this node took an app', async () => { + await appPlacements.recordPlacement('web', june); + + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + }); + + // A redeploy reinstalls an app whose record this node kept. + it('never moves a record that exists', async () => { + await appPlacements.recordPlacement('web', june); + await appPlacements.recordPlacement('web', july); + + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + }); + + it('keeps one record when two writers race the first one', async () => { + await Promise.all([ + appPlacements.recordPlacement('web', june), + appPlacements.recordPlacement('web', july), + appPlacements.recordPlacement('web', june), + ]); + + expect(await recorded()).to.have.length(1); + }); + }); + + describe('placementsOf', () => { + it('answers the apps it has records for, and only those', async () => { + await appPlacements.recordPlacement('web', june); + await appPlacements.recordPlacement('db', july); + + const found = await appPlacements.placementsOf(['web', 'cache']); + + expect([...found]).to.deep.equal([['web', june]]); + }); + }); + + describe('removePlacement', () => { + it('forgets the one app', async () => { + await appPlacements.recordPlacement('web', june); + await appPlacements.recordPlacement('db', july); + + await appPlacements.removePlacement('web'); + + expect(await recorded()).to.deep.equal([{ name: 'db', placedAt: july }]); + }); + }); + + describe('reconcilePlacements', () => { + const row = (name, ip, runningSince) => ({ + name, ip, hash: `${name}hash`, broadcastedAt: new Date(), expireAt: new Date(Date.now() + 3600000), runningSince, + }); + + // Seeding from the row keeps every app where the network already ranks it, so + // a node that has just upgraded does not become the newest copy of all it holds. + it('dates an installed app with no record at the runningSince of this node\'s own row', async () => { + await locations.insertOne(row('web', me, june)); + + const result = await appPlacements.reconcilePlacements(['web'], me); + + expect(result).to.deep.equal({ created: 1, removed: 0 }); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + }); + + it('reads only this node\'s row, never another node\'s', async () => { + await locations.insertOne(row('web', other, june)); + const before = Date.now(); + + await appPlacements.reconcilePlacements(['web'], me); + + const [{ placedAt }] = await recorded(); + expect(placedAt.getTime()).to.be.within(before, Date.now()); + }); + + it('dates an installed app with neither a record nor a row now', async () => { + const before = Date.now(); + + await appPlacements.reconcilePlacements(['web'], me); + + const [{ placedAt }] = await recorded(); + expect(placedAt.getTime()).to.be.within(before, Date.now()); + }); + + it('dates an app now when this node does not know its own address', async () => { + await locations.insertOne(row('web', me, june)); + const before = Date.now(); + + await appPlacements.reconcilePlacements(['web'], null); + + const [{ placedAt }] = await recorded(); + expect(placedAt.getTime()).to.be.within(before, Date.now()); + }); + + it('leaves a record it already has, whatever the row says', async () => { + await appPlacements.recordPlacement('web', june); + await locations.insertOne(row('web', me, july)); + + const result = await appPlacements.reconcilePlacements(['web'], me); + + expect(result).to.deep.equal({ created: 0, removed: 0 }); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + }); + + it('removes the record of an app not installed here', async () => { + await appPlacements.recordPlacement('gone', june); + await appPlacements.recordPlacement('web', july); + + const result = await appPlacements.reconcilePlacements(['web'], me); + + expect(result).to.deep.equal({ created: 0, removed: 1 }); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: july }]); + }); + + it('removes every record when nothing is installed', async () => { + await appPlacements.recordPlacement('gone', june); + + await appPlacements.reconcilePlacements([], me); + + expect(await recorded()).to.deep.equal([]); + }); + }); +}); diff --git a/tests/unit/appRebuild.test.js b/tests/unit/appRebuild.test.js index cb46a8b177..c5f22917ab 100644 --- a/tests/unit/appRebuild.test.js +++ b/tests/unit/appRebuild.test.js @@ -9,6 +9,7 @@ describe('appRebuild tests', () => { let installer; let appRebuild; let bus; + let placements; const spec = { name: 'myapp', version: 8 }; beforeEach(() => { @@ -21,10 +22,12 @@ describe('appRebuild tests', () => { registerAppLocally: sinon.stub().resolves(InstallOutcome.INSTALLED), }; bus = { publish: sinon.stub() }; + placements = { removePlacement: sinon.stub().resolves() }; appRebuild = proxyquire('../../ZelBack/src/services/appLifecycle/appRebuild', { '../utils/fluxEventBus': bus, './appUninstaller': uninstaller, './appInstaller': installer, + '../appManagement/appPlacements': placements, '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, }); }); @@ -40,6 +43,50 @@ describe('appRebuild tests', () => { sinon.assert.notCalled(uninstaller.announceRemoval); }); + // A rebuild removes the app to install it here again, so the record of when + // this node took it stays - unless the app does not come back. + describe('the placement record', () => { + it('is kept through the removal and the reinstall', async () => { + await appRebuild.rebuildApp(spec); + + expect(uninstaller.removeAppLocally.firstCall.args[5]).to.deep.equal({ redeploying: true }); + sinon.assert.notCalled(placements.removePlacement); + }); + + it('goes when the app was removed and not reinstalled', async () => { + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + + await appRebuild.rebuildApp(spec); + + sinon.assert.calledOnceWithExactly(placements.removePlacement, 'myapp'); + }); + + it('goes when the requirements are no longer met', async () => { + installer.checkAppRequirements.rejects(new Error('Insufficient RAM')); + + await appRebuild.rebuildApp(spec); + + sinon.assert.calledOnceWithExactly(placements.removePlacement, 'myapp'); + }); + + it('goes when the app was not installed and did not install', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.NOT_INSTALLED); + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + + await appRebuild.rebuildApp(spec); + + sinon.assert.calledOnceWithExactly(placements.removePlacement, 'myapp'); + }); + + it('stays when the removal did not finish, for the removal that finishes it', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); + + await appRebuild.rebuildApp(spec); + + sinon.assert.notCalled(placements.removePlacement); + }); + }); + it('leaves the installer\'s teardown silent, so the network is told once, here', async () => { await appRebuild.rebuildApp(spec); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 2ed1db2b3f..988517054e 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -17,6 +17,7 @@ describe('appStartupManager tests', () => { let appUninstallerStub; let globalStateStub; let appQueryServiceStub; + let appPlacementsStub; let portManagerStub; let appSliceStub; let nodeStatusMonitor; @@ -82,6 +83,8 @@ describe('appStartupManager tests', () => { sinon.stub(globalStateStub, 'waitForDaemonReady').resolves(); sinon.stub(globalStateStub, 'waitForBootContainerStateSettled').resolves(); + appPlacementsStub = { reconcilePlacements: sinon.stub().resolves({ created: 0, removed: 0 }) }; + appQueryServiceStub = { installedApps: sinon.stub().resolves({ status: 'success', data: [] }), decryptEnterpriseApps: sinon.stub().callsFake(async (apps) => ({ readable: apps, unreadable: [], inPlace: apps })), @@ -109,6 +112,7 @@ describe('appStartupManager tests', () => { './appUninstaller': appUninstallerStub, './appSlice': appSliceStub, '../appMonitoring/nodeStatusMonitor': nodeStatusMonitor, + '../appManagement/appPlacements': appPlacementsStub, '../utils/globalState': globalStateStub, // runs the step again at once until it succeeds, as the real one does after a wait '../utils/untilSucceeds': { @@ -938,7 +942,28 @@ describe('appStartupManager tests', () => { await appStartupManager.manageAppsOnBoot(KEEP); expect(logStub.error.calledWithMatch(/not opened in the firewall: helper crashed/)).to.equal(true); - expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(2); + // once for the placement records, twice for the reconcile + expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(3); + }); + + // The first announcement reads these records, and it waits for boot to settle. + it('makes the placement records match the installed apps before it opens their ports', async () => { + fluxNetworkHelperStub.getLocalSocketAddress.resolves('203.0.113.40:16127'); + + await appStartupManager.manageAppsOnBoot(KEEP); + + sinon.assert.calledOnceWithExactly(appPlacementsStub.reconcilePlacements, ['web', 'db'], '203.0.113.40:16127'); + sinon.assert.callOrder(appPlacementsStub.reconcilePlacements, portManagerStub.appsWithPorts); + }); + + it('logs records it could not reconcile and goes on booting', async () => { + appPlacementsStub.reconcilePlacements.rejects(new Error('mongo went away')); + + await appStartupManager.manageAppsOnBoot(KEEP); + + expect(logStub.error.calledWithMatch(/placement records not reconciled: mongo went away/)).to.equal(true); + expect(logStub.info.calledWithMatch(/node confirmed, reconciling/)).to.equal(true); + expect(globalStateStub.bootContainerStateSettled).to.equal(true); }); it('opens nothing when the apps are all being removed', async () => { diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 959bdc2af2..9317bb8e46 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -7,6 +7,13 @@ const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutc const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); describe('appUninstaller tests', () => { + // The placement record is its own module with its own tests; here it is the + // seam a removal writes through. Fresh per test. + let appPlacementsStub; + beforeEach(() => { + appPlacementsStub = { removePlacement: sinon.stub().resolves() }; + }); + describe('cleanupPorts tests', () => { let appUninstallerPorts; let firewallStub; @@ -19,6 +26,7 @@ describe('appUninstaller tests', () => { }; upnpStub = { isUPNP: sinon.stub().returns(true), removeMapUpnpPort: sinon.stub().resolves(true) }; appUninstallerPorts = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../fluxNetworkHelper': firewallStub, '../upnpService': upnpStub, '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, @@ -117,6 +125,7 @@ describe('appUninstaller tests', () => { runCommandStub = sinon.stub().resolves({ error: null, stdout: '', stderr: '' }); appUninstaller = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -428,6 +437,7 @@ describe('appUninstaller tests', () => { it('should handle app not found case', async () => { const appUninstallerWithDb = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -502,6 +512,7 @@ describe('appUninstaller tests', () => { it('should remove app locally if app name is specified and app in DB', async () => { const appUninstallerWithDbApp = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -629,6 +640,7 @@ describe('appUninstaller tests', () => { }, }; return proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../utils/globalState': globalStateStub, // runs the step again after a moment until it succeeds, as the real one does after its wait '../utils/untilSucceeds': { @@ -1017,6 +1029,54 @@ describe('appUninstaller tests', () => { sinon.assert.notCalled(runtimeStateStub.remove); }); + // When this node took an app is kept for as long as the app stays on it: a + // redeploy removes and reinstalls it without it ever leaving. + describe('the placement record', () => { + it('goes with an app that leaves this node', async () => { + const uninstaller = buildUninstaller(composedSpec); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, true); + + expect(outcome).to.equal(RemovalOutcome.REMOVED); + sinon.assert.calledOnceWithExactly(appPlacementsStub.removePlacement, 'testapp'); + }); + + it('stays when the app is removed to be redeployed here', async () => { + const uninstaller = buildUninstaller(composedSpec); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, false, { redeploying: true }); + + expect(outcome).to.equal(RemovalOutcome.REMOVED); + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + + it('stays when one component is removed and the app with it is not', async () => { + const uninstaller = buildUninstaller(composedSpec); + + await uninstaller.removeAppLocally('comp1_testapp', res, true); + + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + + it('stays through a soft removal', async () => { + const uninstaller = buildUninstaller(composedSpec); + + await uninstaller.softRemoveAppLocally('testapp', null, { removalInProgress: false, installationInProgress: false }, sinon.stub()); + + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + + it('stays when the removal is refused', async () => { + const uninstaller = buildUninstaller(composedSpec); + globalStateStub.removalInProgress = true; + + const outcome = await uninstaller.removeAppLocally('testapp', res, false); + + expect(outcome).to.equal(RemovalOutcome.BUSY); + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + }); + it('completes removal when no seam callback is registered', async () => { const uninstaller = buildUninstaller(v2Spec); @@ -1281,6 +1341,7 @@ describe('appUninstaller tests', () => { db: sinon.stub().returns('appsDatabase'), }; const appUninstallerNoApp = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1365,6 +1426,7 @@ describe('appUninstaller tests', () => { db: sinon.stub().returns('appsDatabase'), }; const appUninstallerWithApp = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index bc7a836e61..a9127a31bc 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -80,6 +80,7 @@ module.exports = { collections: { appsInformation: 'zelappsinformation', appsRuntimeState: 'zelappsruntimestate', + appsPlacements: 'zelappsplacements', }, }, appsglobal: { diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index f433efde0c..5aa39c84a1 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -20,6 +20,7 @@ describe('peerNotification tests', () => { let unconfirmedApps; let announceCycle; let findOneInDatabaseStub; + let placementsOfStub; // One stub map, so a test that needs a different interval, expiry or cycle // length states only that difference instead of restating sixty lines. @@ -115,6 +116,7 @@ describe('peerNotification tests', () => { runningAppsCache: new Set(), }, '../utils/nodeSigner': { nodeSigner: nodeSignerStub }, + '../appManagement/appPlacements': { placementsOf: placementsOfStub }, '../../lib/log': logStub, }); @@ -126,6 +128,8 @@ describe('peerNotification tests', () => { }; findOneInDatabaseStub = sinon.stub().resolves(null); + // every installed app has a record unless a test says otherwise + placementsOfStub = sinon.stub().callsFake(async (names) => new Map(names.map((name) => [name, new Date('2026-09-01T00:00:00.000Z')]))); enqueueAllStub = sinon.stub().resolves(); waitForBootDrainSettledStub = sinon.stub().resolves(); storeAppRunningMessageStub = sinon.stub().resolves(); @@ -359,6 +363,51 @@ describe('peerNotification tests', () => { ).to.deep.equal(['app1']); }); + describe('when this node took each app', () => { + it('announces the time from its placement record, not from its location row', async () => { + placementsOfStub.resolves(new Map([['app1', new Date('2026-06-01T12:00:00.000Z')]])); + findOneInDatabaseStub.resolves({ runningSince: new Date('2026-10-21T02:08:56.000Z') }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps).to.deep.equal([{ name: 'app1', hash: 'app1hash', runningSince: '2026-06-01T12:00:00.000Z' }]); + expect(findOneInDatabaseStub.called, 'the location row is not read for a recorded app').to.equal(false); + }); + + it('keeps announcing the recorded time when its own location row has lapsed', async () => { + placementsOfStub.resolves(new Map([['app1', new Date('2026-06-01T12:00:00.000Z')]])); + findOneInDatabaseStub.resolves(null); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps[0].runningSince).to.equal('2026-06-01T12:00:00.000Z'); + }); + + it('announces an app with no record from its location row, and says so', async () => { + placementsOfStub.resolves(new Map()); + findOneInDatabaseStub.resolves({ runningSince: new Date('2026-07-01T00:00:00.000Z') }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps[0].runningSince).to.equal('2026-07-01T00:00:00.000Z'); + expect(logStub.warn.calledWithMatch(/no placement record for app1/)).to.equal(true); + }); + + it('announces an app with neither a record nor a row as placed now', async () => { + placementsOfStub.resolves(new Map()); + findOneInDatabaseStub.resolves(null); + const before = Date.now(); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(new Date(message.apps[0].runningSince).getTime()).to.be.within(before, Date.now()); + }); + }); + // No node stores an entry without a name and hash, and a node that has not // upgraded refuses the whole message for one - which would cost this node // every other app it holds. From 916cbf62f2cc11ddbf7692190112406d54b0e1c8 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 07:56:52 +0100 Subject: [PATCH 329/369] test(harness): 3501-3503 what a node announces about its apps, and who may place or test one by hand - 3501: an apprunning message is judged one entry at a time, on the node that receives it and on the nodes behind its relay; a message whose every entry is refused keeps the sender's rows, and one naming no apps deletes them; a node holding an app installed without a hash keeps its other apps standing. - 3502: a node announces the time it recorded when it took an app, through its own location rows lapsing and a hard redeploy; at boot the records match the installed apps, a missing one dated at the node's own row; the record goes with the app. - 3503: installapplocally never places an app not yet paid for, for its owner or the flux team; testappinstall tests one for its owner or the flux team, refuses anyone else, and leaves nothing behind. node-client gains testAppInstall and removeAppLocally (always local); db-client gains the placement records and single location-row helpers. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/db-client.js | 39 +++ test-infra/runner/framework/node-client.js | 14 ++ ...ouncement-is-judged-one-entry-at-a-time.js | 229 ++++++++++++++++++ ...s-its-own-record-of-when-it-took-an-app.js | 162 +++++++++++++ ...03-who-may-place-or-test-an-app-by-hand.js | 109 +++++++++ 5 files changed, 553 insertions(+) create mode 100644 test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js create mode 100644 test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js create mode 100644 test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index aba0e5a523..f76f224cfb 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -460,6 +460,45 @@ export function dbClient(nodeNum) { await localDb.collection('zelappsinformation').insertOne(spec); }, + // The app as this node has it installed. + async localApp(name) { + const localDb = await db('appsLocal'); + return localDb.collection('zelappsinformation').findOne({ name }, { projection: { _id: 0 } }); + }, + + // When this node took each app it holds: { name: placedAt }. + async placements() { + const localDb = await db('appsLocal'); + const records = await localDb.collection('zelappsplacements').find({}, { projection: { _id: 0 } }).toArray(); + return Object.fromEntries(records.map(({ name, placedAt }) => [name, placedAt])); + }, + + async seedPlacement(name, placedAt) { + const localDb = await db('appsLocal'); + await localDb.collection('zelappsplacements').insertOne({ name, placedAt: new Date(placedAt) }); + }, + + async deletePlacement(name) { + const localDb = await db('appsLocal'); + await localDb.collection('zelappsplacements').deleteMany({ name }); + }, + + // This node's location row for one app at one address, or null. + async appLocationAt(name, ip) { + const globalDb = await db('appsGlobal'); + return globalDb.collection('zelappslocation').findOne({ name, ip }, { projection: { _id: 0 } }); + }, + + async deleteAppLocation(name, ip) { + const globalDb = await db('appsGlobal'); + await globalDb.collection('zelappslocation').deleteMany({ name, ip }); + }, + + async setAppLocationRunningSince(name, ip, runningSince) { + const globalDb = await db('appsGlobal'); + await globalDb.collection('zelappslocation').updateOne({ name, ip }, { $set: { runningSince: new Date(runningSince) } }); + }, + async seedInstallingLocation({ name, ip, broadcastedAt }) { const globalDb = await db('appsGlobal'); diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 8d9ccd436e..93c6599aa3 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -563,6 +563,20 @@ export function nodeClient(nodeNum) { const res = await controlFetch(`${url}/apps/installapplocally/${appname}`, { headers: { zelidauth } }); return res.text(); }, + // Test an app on THIS node: install it, check it starts, tear it down. By + // name, or by the hash of a temporary message for an app not yet paid for. + // Streams like installapplocally, so drained as text. + testAppInstall: async (appname, zelidauth) => { + const res = await controlFetch(`${url}/apps/testappinstall/${appname}`, { headers: { zelidauth } }); + return res.text(); + }, + // Remove an app from THIS node only. The third path segment is `global`, and + // true would remove the app from every node holding it, so it is always false + // here. Streams progress, so drained as text. + removeAppLocally: async (appname, zelidauth, { force = true } = {}) => { + const res = await controlFetch(`${url}/apps/appremove/${appname}/${force}/false`, { headers: { zelidauth } }); + return res.text(); + }, // Redeploy the WHOLE app on this node, reading the spec from the global table // rather than from what is installed - so a spec seeded since the install is // what the redeploy is judged against. Body is drained as text for the same diff --git a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js new file mode 100644 index 0000000000..e5226e9c92 --- /dev/null +++ b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js @@ -0,0 +1,229 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { waitFor } from '../framework/wait.js'; +import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A version 2 apprunning message names every app its sender holds. Each entry is +// judged on its own: one without a name and a hash is refused and the rest of the +// message is stored and relayed as if it were not there. +// +// The stub peer is a listed node with its own key, peered with node 0 only, so +// nodes 1 and 2 hear what it says only through node 0's relay. A message node 0 +// does not store goes no further, so the relay is part of what is asserted. +// +// The last case is a real node that holds an app installed without a hash beside +// one installed normally: its announcements keep the normal app's rows standing +// on every node. + +const REAL_NODES = 3; +const STUB_INDEX = 3; +// The harness's own location lifetime (63s), so a row that is not refreshed is +// gone within a minute and a node that went quiet is visible as one. +const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); + +function appComponent(appName) { + return { + name: 'alpha', + description: 'one entry at a time', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +describe('an announcement is judged one entry at a time', function () { + let env; + dumpLogsOnFailure(() => env); + let stub; + const stamp = Date.now(); + const stubAddress = () => socketAddr(STUB_INDEX + 1); + const rowNamesAt = async (i, address) => (await dbClient(i + 1).getAppLocationsByIp(address)).map((row) => row.name); + const everyNodeHolds = (address, name) => async () => (await Promise.all(realNodes.map((i) => rowNamesAt(i, address)))) + .every((names) => names.includes(name)); + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ + hookCtx: this, + nodes: REAL_NODES + 1, + stubPeers: [STUB_INDEX], + stubPeeredWith: { [STUB_INDEX]: [0] }, + tickerAutostart: false, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + stub = env.stubPeerClients.get(STUB_INDEX); + await waitFor(async () => (await stub.connectedNodes()) >= 1, { + timeout: 120000, interval: 2000, label: 'node 0 is connected to the stub peer', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + // Nodes 1 and 2 must hear the stub only through node 0, or the relay is not + // what carried the message to them. + async function assertOnlyNodeZeroHearsTheStub() { + const stubIp = stubAddress().split(':')[0]; + // eslint-disable-next-line no-restricted-syntax + for (const i of [1, 2]) { + // eslint-disable-next-line no-await-in-loop + const [outgoing, incoming] = await Promise.all([env.clients[i].getPeers(), env.clients[i].getIncomingPeers()]); + const peers = JSON.stringify([...(outgoing.data ?? []), ...(incoming.data ?? [])]); + expect(peers.includes(stubIp), `fixture: node ${i} is peered with the stub directly`).to.equal(false); + } + } + + it('stores and relays the entries with a name and hash, and refuses the one without', async function () { + this.timeout(120000); + await assertOnlyNodeZeroHearsTheStub(); + const kept = `e2eentrykept${stamp}`; + const refused = `e2eentryrefused${stamp}`; + + const sent = await stub.broadcast({ + type: 'fluxapprunning', + version: 2, + apps: [ + { name: kept, hash: `hash-${kept}`, runningSince: new Date().toISOString() }, + { name: refused, runningSince: new Date().toISOString() }, + ], + ip: stubAddress(), + broadcastedAt: Date.now(), + osUptime: 10000, + staticIp: false, + }); + expect(sent.sent, 'fixture: the message reached node 0').to.equal(1); + + await waitFor(everyNodeHolds(stubAddress(), kept), { + timeout: 60000, interval: 2000, label: 'every node, the two behind node 0 included, holds the entry with a hash', + }); + // eslint-disable-next-line no-restricted-syntax + for (const i of realNodes) { + // eslint-disable-next-line no-await-in-loop + expect(await rowNamesAt(i, stubAddress()), `node ${i} stored the entry without a hash`).to.not.include(refused); + } + }); + + // An empty v2 message deletes the sender's rows. A message whose entries were + // all refused named apps, and is never read as one that named none. + it('keeps the sender\'s rows when every entry of a message is refused', async function () { + this.timeout(120000); + const held = `e2eentryheld${stamp}`; + const marker = `e2eentrymarker${stamp}`; + const heldAt = Date.now(); + await stub.runApp(held, { hash: `hash-${held}`, broadcastedAt: heldAt }); + await waitFor(everyNodeHolds(stubAddress(), held), { + timeout: 60000, interval: 2000, label: 'every node holds the stub peer\'s app', + }); + + await stub.broadcast({ + type: 'fluxapprunning', + version: 2, + apps: [{ name: `e2eentryrefusedonly${stamp}` }], + ip: stubAddress(), + broadcastedAt: Date.now(), + osUptime: 10000, + staticIp: false, + }); + // A node handles one peer's messages in the order they were sent, so node 0 + // holding the marker has handled the refused message before it. + await stub.runApp(marker, { hash: `hash-${marker}` }); + await waitFor(everyNodeHolds(stubAddress(), marker), { + timeout: 60000, interval: 2000, label: 'every node holds the message sent after the refused one', + }); + + expect(Date.now() - heldAt, 'fixture: the app\'s rows have not reached their own expiry').to.be.below(50000); + // eslint-disable-next-line no-restricted-syntax + for (const i of realNodes) { + // eslint-disable-next-line no-await-in-loop + expect(await rowNamesAt(i, stubAddress()), `node ${i} deleted the sender's rows for a message whose entries it refused`).to.include(held); + } + }); + + // What the case above guards against is real: a message naming no apps does + // delete the sender's rows, on node 0 and on the nodes behind it. + it('deletes the sender\'s rows on a message that names no apps', async function () { + this.timeout(120000); + const held = `e2eentrycleared${stamp}`; + await stub.runApp(held, { hash: `hash-${held}` }); + await waitFor(everyNodeHolds(stubAddress(), held), { + timeout: 60000, interval: 2000, label: 'every node holds the stub peer\'s app', + }); + + await stub.broadcast({ + type: 'fluxapprunning', + version: 2, + apps: [], + ip: stubAddress(), + broadcastedAt: Date.now(), + osUptime: 10000, + staticIp: false, + }); + + await waitFor(async () => (await Promise.all(realNodes.map((i) => rowNamesAt(i, stubAddress())))).every((names) => names.length === 0), { + timeout: 60000, interval: 2000, label: 'every node dropped the stub peer\'s rows', + }); + }); + + it('keeps a node\'s apps standing while it holds an app installed without a hash', async function () { + this.timeout(420000); + const appName = `e2eentryreal${stamp}`; + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ name: appName, instances: 1, compose: [appComponent(appName)] }); + const idx = await seedAndInstall(env, app); + const holder = socketAddr(idx + 1); + const others = realNodes.filter((i) => i !== idx); + await waitFor(everyNodeHolds(holder, appName), { + timeout: 180000, interval: 3000, label: 'every node holds the app at its holder', + }); + + // An app installed without a hash, as a node carries one it took before it + // upgraded. Everything else about it is the real app's. + const unhashed = `e2eentryunhashed${stamp}`; + const installed = await dbClient(idx + 1).localApp(appName); + const { hash, ...withoutHash } = installed; + expect(hash, 'fixture: the real app is installed with a hash').to.be.a('string'); + const seededAt = Date.now(); + await dbClient(idx + 1).seedLocalApp({ ...withoutHash, name: unhashed }); + + const mark = env.clients[idx].getLastEventId(); + const announced = await env.clients[idx].waitForEvent('app:running', (d) => d.apps?.some((a) => a.name === appName), 120000, { afterId: mark }); + expect(announced.apps.map((a) => a.name), 'the holder announced the app it has no hash for').to.not.include(unhashed); + expect(await dbClient(idx + 1).localApp(unhashed), 'fixture: the unhashed app was still installed when the holder announced').to.not.equal(null); + + // Two location lifetimes after the seeding, the app's rows on every other node + // are ones written since: a holder whose announcements were refused would have + // let them lapse. + await waitFor(async () => Date.now() - seededAt > 130000, { timeout: 140000, interval: 5000, label: 'two location lifetimes pass' }); + // eslint-disable-next-line no-restricted-syntax + for (const i of others) { + // eslint-disable-next-line no-await-in-loop + const row = await dbClient(i + 1).appLocationAt(appName, holder); + expect(row, `node ${i} lost the holder's row`).to.not.equal(null); + expect(new Date(row.broadcastedAt).getTime(), `node ${i} holds no announcement made since`).to.be.above(seededAt); + } + const own = await dbClient(idx + 1).appLocationAt(appName, holder); + expect(own, 'the holder lost its own row').to.not.equal(null); + expect(new Date(own.broadcastedAt).getTime(), 'the holder stored none of its own announcements').to.be.above(seededAt); + }); +}); diff --git a/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js new file mode 100644 index 0000000000..fbb81217da --- /dev/null +++ b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js @@ -0,0 +1,162 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { restartFluxos } from '../framework/container.js'; +import { socketAddr } from '../framework/state-events.js'; +import { fluxTeamKey } from '../framework/keys.js'; +import { authenticate } from '../auth.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node records when it took an app, once, and announces that as the app's +// runningSince for as long as the app stays on it. The network's copy of the +// node's own location row is not where it comes from: that row lapses while the +// app goes on running, and a redeploy removes and reinstalls the app without it +// leaving. The record goes with the app, and at boot the records are made to +// match what is installed, dating any missing one at the node's own row. + +const REAL_NODES = 3; +const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); +const DAY_MS = 24 * 60 * 60 * 1000; + +describe('a node keeps its own record of when it took an app', function () { + let env; + dumpLogsOnFailure(() => env); + let idx; + let holder; + let others; + let placedAt; + const appName = `e2eplacement${Date.now()}`; + + const placementOf = async (name) => (await dbClient(idx + 1).placements())[name]; + // The runningSince every other node holds for the holder, once each has stored + // an announcement made after `since`. + async function announcedSince(since) { + const rows = await Promise.all(others.map((i) => dbClient(i + 1).appLocationAt(appName, holder))); + if (!rows.every((row) => row && new Date(row.broadcastedAt).getTime() > since)) return null; + return rows.map((row) => new Date(row.runningSince).getTime()); + } + async function waitForAnnouncedSince(since, label) { + let seen = null; + await waitFor(async () => { seen = await announcedSince(since); return seen !== null; }, { + timeout: 120000, interval: 3000, label, + }); + return seen; + } + const teamAuth = async () => (await authenticate(env.clients[idx].url, fluxTeamKey())).zelidauth; + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ hookCtx: this, nodes: REAL_NODES, tickerAutostart: false }); + await bootAndPeer(env); + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ + name: appName, + instances: 1, + compose: [{ + name: 'alpha', + description: 'placement record', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + idx = await seedAndInstall(env, app); + holder = socketAddr(idx + 1); + others = realNodes.filter((i) => i !== idx); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('records when it took the app, and announces that', async function () { + this.timeout(180000); + const recorded = await placementOf(appName); + expect(recorded, 'no record of when the holder took the app').to.be.instanceOf(Date); + placedAt = recorded.getTime(); + + const seen = await waitForAnnouncedSince(placedAt, 'every other node holds an announcement made since the install'); + expect(seen, 'the announced runningSince is not the recorded one').to.deep.equal(others.map(() => placedAt)); + }); + + it('keeps announcing the recorded time after its own location rows lapse', async function () { + this.timeout(180000); + // Every node's row for the holder, the holder's own included, as after an + // expiry, an eviction or a sigterm. + await Promise.all(realNodes.map((i) => dbClient(i + 1).deleteAppLocation(appName, holder))); + const lapsedAt = Date.now(); + + const seen = await waitForAnnouncedSince(lapsedAt, 'every other node holds an announcement made since the rows lapsed'); + expect(seen, 'the holder re-dated the app when its row lapsed').to.deep.equal(others.map(() => placedAt)); + expect((await placementOf(appName)).getTime(), 'the record moved').to.equal(placedAt); + }); + + it('keeps the record through a hard redeploy', async function () { + this.timeout(300000); + const mark = env.clients[idx].getLastEventId(); + const body = await env.clients[idx].redeployApp(appName, await teamAuth(), { force: true }); + const rebuilt = await env.clients[idx].waitForEvent('app:rebuilt', (d) => d.name === appName, 240000, { afterId: mark }); + expect(rebuilt.outcome, `the redeploy did not reinstall the app: ${body.slice(-400)}`).to.equal('installed'); + const redeployedAt = Date.now(); + + expect((await placementOf(appName)).getTime(), 'the redeploy moved the record').to.equal(placedAt); + const seen = await waitForAnnouncedSince(redeployedAt, 'every other node holds an announcement made since the redeploy'); + expect(seen, 'the redeploy re-dated the app').to.deep.equal(others.map(() => placedAt)); + }); + + // A node that took an app before it had these records gets one at boot, dated + // at the runningSince its own row already carries; a record for an app it does + // not hold is dropped. + it('makes its records match its installed apps at boot, dating a missing one at its own row', async function () { + this.timeout(420000); + const rowSince = Date.now() - 3 * DAY_MS; + const ghost = `e2eplacementghost${Date.now()}`; + await dbClient(idx + 1).deletePlacement(appName); + await dbClient(idx + 1).seedPlacement(ghost, Date.now() - DAY_MS); + await dbClient(idx + 1).setAppLocationRunningSince(appName, holder, rowSince); + + // FluxOS alone, so the downtime is seconds and the node keeps its apps. + const mark = env.clients[idx].getLastEventId(); + await restartFluxos(env.clients[idx].container); + await waitForBootSettled(env.clients[idx], 240000, { afterId: mark }); + + const records = await dbClient(idx + 1).placements(); + expect(records[appName]?.getTime(), 'the missing record was not dated at the node\'s own row').to.equal(rowSince); + expect(records, 'the record of an app the node does not hold survived boot').to.not.have.property(ghost); + const bootedAt = Date.now(); + const seen = await waitForAnnouncedSince(bootedAt, 'every other node holds an announcement made since the boot'); + expect(seen, 'the announced runningSince is not the recorded one').to.deep.equal(others.map(() => rowSince)); + placedAt = rowSince; + }); + + it('drops the record with the app, so a later install starts a new one', async function () { + this.timeout(240000); + const mark = env.clients[idx].getLastEventId(); + const body = await env.clients[idx].removeAppLocally(appName, await teamAuth()); + await env.clients[idx].waitForEvent('app:removed', (d) => d.name === appName, 180000, { afterId: mark }); + + // The node's own spawner may take the app back at once; whichever record + // stands is a new one, never the one the removal ended. + const after = await placementOf(appName); + expect(after?.getTime(), `the removal kept the record: ${body.slice(-400)}`).to.not.equal(placedAt); + if (after) expect(after.getTime(), 'a record older than the removal').to.be.at.least(Date.now() - 180000); + }); +}); diff --git a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js new file mode 100644 index 0000000000..948ba78d7c --- /dev/null +++ b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js @@ -0,0 +1,109 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { buildAppSpec, registerApp } from '../framework/app-helper.js'; +import { + appOwnerKey, fluxTeamKey, nodeKey, userKey, +} from '../framework/keys.js'; +import { authenticate } from '../auth.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app registered and not yet paid for exists only as a temporary message. +// installapplocally places registered apps, for the flux team, and never one of +// these. testappinstall tries one out - installs it, checks it starts, tears it +// down, and never announces it - for its owner or the flux team, and refuses any +// other caller before a container is made. + +const REAL_NODES = 2; +const NODE = 0; + +describe('who may place or test an app by hand', function () { + let env; + dumpLogsOnFailure(() => env); + let appName; + let messageHash; + const client = () => env.clients[NODE]; + const authAs = async (key) => (await authenticate(client().url, key)).zelidauth; + + // Nothing of the app is left on the node, and no node holds a location for it there. + async function assertNothingHeld() { + expect(await dbClient(NODE + 1).localApp(appName), 'the app is installed on the node').to.equal(null); + expect(await dbClient(NODE + 1).placements(), 'the node holds a placement record for the app').to.not.have.property(appName); + const rows = await Promise.all(env.clients.map((_c, i) => dbClient(i + 1).appLocationAt(appName, socketAddr(NODE + 1)))); + expect(rows.filter(Boolean), 'a node holds a location for the app at the node').to.deep.equal([]); + } + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ hookCtx: this, nodes: REAL_NODES, tickerAutostart: false }); + await bootAndPeer(env); + await Promise.all(env.clients.map((c) => waitForBootSettled(c, 240000))); + + appName = `e2ehandplaced${Date.now()}`; + const spec = buildAppSpec({ name: appName }); + const result = await registerApp(client().url, nodeKey(1), spec); + expect(result.status, `fixture: the registration was refused: ${JSON.stringify(result)}`).to.equal('success'); + messageHash = result.data; + await waitFor(async () => (await client().getTempMessages(messageHash)).data?.length > 0, { + timeout: 60000, interval: 2000, label: 'the node holds the temporary message', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('refuses its owner placing it with installapplocally', async function () { + this.timeout(120000); + const body = await client().installAppLocally(messageHash, await authAs(appOwnerKey())); + + expect(body, 'the owner was not refused').to.match(/Unauthorized/); + await assertNothingHeld(); + }); + + it('does not place it for the flux team either', async function () { + this.timeout(120000); + const body = await client().installAppLocally(messageHash, await authAs(fluxTeamKey())); + + expect(body, 'installapplocally read the temporary message').to.match(new RegExp(`Application Specifications of ${messageHash} not found`)); + await assertNothingHeld(); + }); + + it('refuses another user testing it', async function () { + this.timeout(120000); + const body = await client().testAppInstall(messageHash, await authAs(userKey())); + + expect(body, 'another user was not refused').to.match(/Unauthorized/); + await assertNothingHeld(); + }); + + it('lets its owner test it, and leaves nothing behind', async function () { + this.timeout(300000); + const body = await client().testAppInstall(messageHash, await authAs(appOwnerKey())); + + expect(body, 'the owner\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); + await waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { + timeout: 60000, interval: 2000, label: 'the test install is torn down', + }); + await assertNothingHeld(); + }); + + it('lets the flux team test it', async function () { + this.timeout(300000); + const body = await client().testAppInstall(messageHash, await authAs(fluxTeamKey())); + + expect(body, 'the flux team\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); + await waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { + timeout: 60000, interval: 2000, label: 'the test install is torn down', + }); + await assertNothingHeld(); + }); +}); From 0a0c55c0c48594a587476440ce73964608827739 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:00:38 +0100 Subject: [PATCH 330/369] fix(apps): an apprunning message naming only refused entries is not a snapshot anywhere The event log a node serves to peers syncing from it keeps the sender's last announcement over a later one whose every entry it refuses, on the live and the sync path alike. Peer sync prunes a node's rows against the entries its newest readable broadcast can store, never against one naming only refused entries. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMessaging/messageStore.js | 14 ++++++ ZelBack/src/services/fluxCommunication.js | 8 ++- tests/unit/fluxCommunication.test.js | 49 +++++++++++++++++++ .../messageStoreAppRunningEntries.test.js | 40 +++++++++++++++ 4 files changed, 110 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index e0c9af24e9..120da56a56 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -906,10 +906,23 @@ async function pruneAppRunningLocations(newestByIp) { // --- Event Log Functions --- +/** + * Whether an apprunning message names apps and none of them can be stored. Such a + * message says nothing this node can act on, and is kept out of the event log so + * it never takes the place of the sender's last announcement there. + * @param {object} data The message's data. + * @returns {boolean} + */ +function namesOnlyRefusedApps(data) { + return data.version === 2 && Array.isArray(data.apps) && data.apps.length > 0 + && appRunningEntries(data.apps).stored.length === 0; +} + async function handleAppRunningEvent({ signedBroadcast }) { const { data } = signedBroadcast; if (!data || !data.ip || !data.broadcastedAt) return; if (data.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) return; + if (namesOnlyRefusedApps(data)) return; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -1072,6 +1085,7 @@ async function storeBatchAppRunningEvents(verifiedBroadcasts) { if (!data || !data.ip || !data.broadcastedAt) continue; const validTill = data.broadcastedAt + RUNNING_EXPIRY_MS; if (validTill < Date.now()) continue; + if (namesOnlyRefusedApps(data)) continue; const dedupKey = data.apps ? 'v2' : `v1:${data.name}`; const envelope = { version: broadcast.version, timestamp: broadcast.timestamp, pubKey: broadcast.pubKey, signature: broadcast.signature }; diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 37415f7c89..2f678fb3f2 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -6,6 +6,7 @@ const log = require('../lib/log'); const serviceHelper = require('./serviceHelper'); const dbHelper = require('./dbHelper'); const messageStore = require('./appMessaging/messageStore'); +const { appRunningEntries } = require('./utils/appRunningEntries'); const verificationHelper = require('./verificationHelper'); const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); const fluxCommunicationMessagesSender = require('./fluxCommunicationMessagesSender'); @@ -262,9 +263,14 @@ async function handleAppRunningSyncResponse(message, peerSocket) { // already-expired rows - but that is a bound to be read out of another // file, and two builders of one input should not need reconciling. if (data.broadcastedAt + RUNNING_EXPIRY_MS < Date.now()) continue; + // Pruned against the apps the broadcast names that can be stored: one + // naming only entries this node refuses says nothing about what its + // sender runs, and is never read as one naming no apps at all. + const { stored } = appRunningEntries(data.apps); + if (!stored.length) continue; const seen = newestByIp.get(data.ip); if (!seen || data.broadcastedAt > seen.broadcastedAt) { - newestByIp.set(data.ip, { names: data.apps.map((a) => a.name), broadcastedAt: data.broadcastedAt }); + newestByIp.set(data.ip, { names: stored.map((a) => a.name), broadcastedAt: data.broadcastedAt }); } } diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index 2745bd7f33..f03127bb21 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -2324,6 +2324,55 @@ describe('fluxCommunication tests', () => { sinon.assert.notCalled(messageStore.storeIPChangedMessage); }); + // A node's newest broadcast says which of its rows stand; its refused + // entries say nothing about what it runs. + describe('pruning against what a synced node runs', () => { + const apprunning = (apps, broadcastedAt = Date.now() - 60000) => ({ + type: 'apprunning', + ip: SENDER, + envelope, + data: { + type: 'fluxapprunning', version: 2, apps, ip: SENDER, broadcastedAt, osUptime: 1, staticIp: false, + }, + }); + + beforeEach(() => { + sinon.stub(messageStore, 'storeBatchAppRunningMessages').resolves({ stored: 1, writeFailed: false }); + }); + + it('prunes against the apps the broadcast names that can be stored', async () => { + await fluxCommunication.handleAppRunningSyncResponse({ + data: { type: 'fluxapprunningsync', messages: [apprunning([{ name: 'web', hash: 'h1' }, { name: 'probe' }])], done: true }, + }, PEER_SOCKET); + + const [newestByIp] = messageStore.pruneAppRunningLocations.firstCall.args; + expect(newestByIp.get(SENDER).names).to.deep.equal(['web']); + }); + + it('never prunes against a broadcast whose every app is refused', async () => { + await fluxCommunication.handleAppRunningSyncResponse({ + data: { type: 'fluxapprunningsync', messages: [apprunning([{ name: 'probe' }])], done: true }, + }, PEER_SOCKET); + + const [newestByIp] = messageStore.pruneAppRunningLocations.firstCall.args; + expect(newestByIp.has(SENDER), 'pruned the sender\'s rows against an empty list').to.equal(false); + }); + + it('prunes against the newest broadcast it can read, past a later one it refuses', async () => { + const now = Date.now(); + await fluxCommunication.handleAppRunningSyncResponse({ + data: { + type: 'fluxapprunningsync', + messages: [apprunning([{ name: 'web', hash: 'h1' }], now - 120000), apprunning([{ name: 'probe' }], now - 60000)], + done: true, + }, + }, PEER_SOCKET); + + const [newestByIp] = messageStore.pruneAppRunningLocations.firstCall.args; + expect(newestByIp.get(SENDER)).to.deep.equal({ names: ['web'], broadcastedAt: now - 120000 }); + }); + }); + it('hands an eviction to messageStore with the time it was made', async () => { const evictStub = sinon.stub(messageStore, 'applyEviction').resolves(); const createdAt = new Date(Date.now() - 45 * 60 * 1000); diff --git a/tests/unit/messageStoreAppRunningEntries.test.js b/tests/unit/messageStoreAppRunningEntries.test.js index 49da92a784..e12f0c6647 100644 --- a/tests/unit/messageStoreAppRunningEntries.test.js +++ b/tests/unit/messageStoreAppRunningEntries.test.js @@ -150,6 +150,46 @@ describe('apprunning entries are stored one at a time', () => { }); }); + // The event log is what a peer syncing from this node is served. A message + // naming only entries this node refuses never takes the place of the + // sender's last announcement there. + describe('the event log', () => { + let events; + const signed = (apps, broadcastedAt = Date.now()) => ({ + version: 1, timestamp: broadcastedAt, pubKey: '04ab', signature: 'sig', data: message(apps, broadcastedAt), + }); + const storedApps = async () => (await events.find({ ip, type: 'apprunning' }).toArray()).map((event) => event.data.apps.map((a) => a.name)); + + beforeEach(async () => { + events = dbHelper.databaseConnection().db(config.database.appsglobal.database) + .collection(config.database.appsglobal.collections.appStateEvents); + await events.deleteMany({ ip }); + }); + + afterEach(async () => { + await events.deleteMany({ ip }); + }); + + it('keeps the sender\'s last announcement over a later one naming only refused entries', async () => { + await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed([{ name: 'web', hash: 'webhash' }], Date.now() - 1000) }); + await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed([{ name: 'probe' }]) }); + + expect(await storedApps()).to.deep.equal([['web']]); + }); + + it('records a mixed announcement whole, as it was signed', async () => { + await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed([{ name: 'web', hash: 'webhash' }, { name: 'probe' }]) }); + + expect(await storedApps()).to.deep.equal([['web', 'probe']]); + }); + + it('keeps a synced announcement naming only refused entries out of the log', async () => { + await messageStore.storeBatchAppRunningEvents([signed([{ name: 'probe' }])]); + + expect(await storedApps()).to.deep.equal([]); + }); + }); + describe('storeBatchAppRunningMessages', () => { const broadcast = (data) => ({ version: 1, timestamp: data.broadcastedAt, pubKey: '04ab', signature: 'sig', data }); From fb74fa3bed3b4917870873654518fa61cad0431a Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:02:02 +0100 Subject: [PATCH 331/369] test(harness): 3503 registers with the peers a two-node fleet holds Co-Authored-By: Claude Opus 5.5 --- .../tests/3503-who-may-place-or-test-an-app-by-hand.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js index 948ba78d7c..9a494cac7e 100644 --- a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js +++ b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js @@ -42,8 +42,14 @@ describe('who may place or test an app by hand', function () { before(async function () { this.timeout(420000); - env = await createTestEnv({ hookCtx: this, nodes: REAL_NODES, tickerAutostart: false }); - await bootAndPeer(env); + // Registration wants this many peers each way, and two nodes hold one. + env = await createTestEnv({ + hookCtx: this, + nodes: REAL_NODES, + tickerAutostart: false, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); await Promise.all(env.clients.map((c) => waitForBootSettled(c, 240000))); appName = `e2ehandplaced${Date.now()}`; From 27fb59dadcc68f861b7e23e55f850a84dfa6117c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:10:28 +0100 Subject: [PATCH 332/369] feat(harness): announcement refusals and placement records are published as events network:apprunningRefused names the entries a node refused, by gossip or by sync. app:placementRecorded, app:placementRemoved and app:placementsReconciled mark each write and removal of a placement record and the boot step. app:running carries the installed apps the announcement left out, and app:announcementRefused marks an announcement the node's own store refused. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appManagement/appPlacements.js | 15 +++++++++++---- ZelBack/src/services/appMessaging/messageStore.js | 12 +++++++++++- .../src/services/appMessaging/peerNotification.js | 3 ++- tests/unit/messageStore.test.js | 2 +- 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js index 833e3cc1ca..73740afed7 100644 --- a/ZelBack/src/services/appManagement/appPlacements.js +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -1,6 +1,7 @@ const config = require('config'); const log = require('../../lib/log'); const dbHelper = require('../dbHelper'); +const fluxEventBus = require('../utils/fluxEventBus'); // When this node took each app it holds: one record per app, keyed by name. // @@ -35,18 +36,22 @@ async function prepareCollection() { * Record that this node took an app at `placedAt`, unless a record for it exists. * @param {string} name App name. * @param {Date} [placedAt] When this node took it. + * @param {string} [source] What the time was taken from: install, locationRow or now. * @returns {Promise} */ -async function recordPlacement(name, placedAt = new Date()) { +async function recordPlacement(name, placedAt = new Date(), source = 'install') { // Two writers racing the first insert leave one record: the server retries an // upsert that collides on a unique index matched by equality. - await dbHelper.updateOneInDatabase( + const result = await dbHelper.updateOneInDatabase( localDatabase(), appsPlacements, { name }, { $setOnInsert: { name, placedAt } }, { upsert: true }, ); + if (result?.upsertedCount) { + fluxEventBus.publish('app:placementRecorded', { name, placedAt: placedAt.toISOString(), source }); + } } /** @@ -70,7 +75,8 @@ async function placementsOf(names) { * @returns {Promise} */ async function removePlacement(name) { - await dbHelper.removeDocumentsFromCollection(localDatabase(), appsPlacements, { name }); + const result = await dbHelper.removeDocumentsFromCollection(localDatabase(), appsPlacements, { name }); + if (result?.deletedCount) fluxEventBus.publish('app:placementRemoved', { name }); } /** @@ -103,7 +109,7 @@ async function reconcilePlacements(installedNames, localSocketAddr) { // eslint-disable-next-line no-restricted-syntax for (const name of missing) { // eslint-disable-next-line no-await-in-loop - await recordPlacement(name, rowSince.get(name) ?? now); + await recordPlacement(name, rowSince.get(name) ?? now, rowSince.has(name) ? 'locationRow' : 'now'); } const removal = await dbHelper.removeDocumentsFromCollection(database, appsPlacements, { name: { $nin: installedNames } }); @@ -111,6 +117,7 @@ async function reconcilePlacements(installedNames, localSocketAddr) { if (missing.length || removed) { log.info(`appPlacements - recorded ${missing.length} placement(s) (${rowSince.size} from this node's location rows), removed ${removed} for apps not installed here`); } + fluxEventBus.publish('app:placementsReconciled', { created: missing.length, fromLocationRows: rowSince.size, removed }); return { created: missing.length, removed }; } diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index 120da56a56..46a444c22c 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -11,6 +11,7 @@ const { socketAddressesMatch } = require('../utils/socketAddressUtils'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const globalState = require('../utils/globalState'); const { appRunningEntries } = require('../utils/appRunningEntries'); +const fluxEventBus = require('../utils/fluxEventBus'); const { globalAppsMessages, globalAppsTempMessages, @@ -356,7 +357,10 @@ async function storeAppRunningMessage(message) { return new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode'); } const { stored, refused } = appRunningEntries(message.apps); - if (refused.length) logRefusedEntries('storeAppRunningMessage', [{ ip: message.ip, broadcastedAt: message.broadcastedAt, refused }]); + if (refused.length) { + logRefusedEntries('storeAppRunningMessage', [{ ip: message.ip, broadcastedAt: message.broadcastedAt, refused }]); + fluxEventBus.publish('network:apprunningRefused', { path: 'gossip', refusals: [{ ip: message.ip, names: refused.map((app) => app?.name ?? null) }] }); + } if (message.apps.length && !stored.length) { return { stored: false, rebroadcast: false }; } @@ -854,6 +858,12 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { await flushLocationOps(); logRefusedEntries('storeBatchAppRunningMessages', refusals); + if (refusals.length) { + fluxEventBus.publish('network:apprunningRefused', { + path: 'sync', + refusals: refusals.map(({ ip, refused }) => ({ ip, names: refused.map((app) => app?.name ?? null) })), + }); + } // Upserts only. Removing the rows a node no longer reports belongs to // pruneAppRunningLocations, which is driven from the newest broadcast per node diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index fe1432de8e..545898602e 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -268,11 +268,12 @@ async function checkAndNotifyPeersOfRunningApps() { const ownStore = await messageStore.storeAppRunningMessage(appRunningMessage); if (ownStore instanceof Error) { log.error(`checkAndNotifyPeersOfRunningApps - this node refuses its own announcement, not broadcast: ${ownStore.message}`); + fluxEventBus.publish('app:announcementRefused', { reason: ownStore.message }); return; } const signed = await fluxCommunicationMessagesSender.broadcastMessageToAll(appRunningMessage); await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed }); - fluxEventBus.publish('app:running', { apps, ip: appRunningMessage.ip }); + fluxEventBus.publish('app:running', { apps, ip: appRunningMessage.ip, unannounced: refused.map((application) => application?.name ?? null) }); log.info(`App Running Message broadcasted: ${apps.length} apps`); } catch (err) { log.error(err); diff --git a/tests/unit/messageStore.test.js b/tests/unit/messageStore.test.js index a038555281..3cee0f9ef8 100644 --- a/tests/unit/messageStore.test.js +++ b/tests/unit/messageStore.test.js @@ -1115,7 +1115,7 @@ describe('messageStore tests', () => { const now = () => Date.now(); // Every event a node records once and serves to the peers syncing from it. const events = () => [ - ['apprunning', { signedBroadcast: { data: { ip: '1.2.3.4:16127', broadcastedAt: now(), version: 2, apps: [{ name: 'a' }] } } }], + ['apprunning', { signedBroadcast: { data: { ip: '1.2.3.4:16127', broadcastedAt: now(), version: 2, apps: [{ name: 'a', hash: 'h' }] } } }], ['sigterm', { message: { ip: '1.2.3.4:16127', broadcastedAt: now() } }], ['appremoved', { message: { ip: '1.2.3.4:16127', appName: 'a', broadcastedAt: now() } }], ['evicted', { ip: '1.2.3.4:16127', evictedAt: now() }], From b27baab6a71823d3783f2180e282190fb339e1d9 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:10:29 +0100 Subject: [PATCH 333/369] test(harness): 3501-3503 assert on the node's events The suites wait on what each node publishes - the entries it stored and refused, the placement records it wrote and removed, what it announced - and assert the refusals and non-relays with assertNoEvent. 3501 cuts nodes 1 and 2 off from the stub peer, so they hear it only through node 0's relay. The new events are on the client's whitelist. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/node-client.js | 7 + ...ouncement-is-judged-one-entry-at-a-time.js | 214 +++++++++--------- ...s-its-own-record-of-when-it-took-an-app.js | 120 +++++----- ...03-who-may-place-or-test-an-app-by-hand.js | 33 +-- 4 files changed, 191 insertions(+), 183 deletions(-) diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 93c6599aa3..9e26eee072 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -226,6 +226,11 @@ export function nodeClient(nodeNum) { 'app:componentRedeployed', 'app:specStored', 'app:running', + // what this node records of when it took each app it holds + 'app:placementRecorded', + 'app:placementRemoved', + 'app:placementsReconciled', + 'app:announcementRefused', 'app:announcing', 'fileoperation:imageAcquired', 'fileoperation:imageDiscarded', @@ -317,6 +322,8 @@ export function nodeClient(nodeNum) { 'hostStall:appUnloaded', 'hostStall:appsStopped', 'network:apprunning', + // entries of an apprunning message this node refused, by gossip or sync + 'network:apprunningRefused', 'network:appinstalling', 'network:appinstallingerror', 'network:appremoved', diff --git a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js index e5226e9c92..6d68a2767a 100644 --- a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js +++ b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js @@ -6,30 +6,33 @@ import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; -import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { waitFor } from '../framework/wait.js'; +import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, assertNoEvent } from '../framework/wait.js'; import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; +import { execInContainer } from '../framework/container.js'; import { socketAddr } from '../framework/state-events.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // A version 2 apprunning message names every app its sender holds. Each entry is -// judged on its own: one without a name and a hash is refused and the rest of the -// message is stored and relayed as if it were not there. +// judged on its own: one without a name and a hash is refused +// (network:apprunningRefused) and the rest of the message is stored +// (network:apprunning) and relayed as if it were not there. // -// The stub peer is a listed node with its own key, peered with node 0 only, so -// nodes 1 and 2 hear what it says only through node 0's relay. A message node 0 -// does not store goes no further, so the relay is part of what is asserted. +// The stub peer is a listed node with its own key. Nodes 1 and 2 cannot reach it, +// so they hear what it says only through node 0's relay, and a message node 0 +// does not store goes no further. // // The last case is a real node that holds an app installed without a hash beside -// one installed normally: its announcements keep the normal app's rows standing -// on every node. +// one installed normally: it leaves the first out of its announcements +// (app:running.unannounced), and every node goes on storing them. const REAL_NODES = 3; const STUB_INDEX = 3; -// The harness's own location lifetime (63s), so a row that is not refreshed is -// gone within a minute and a node that went quiet is visible as one. const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); +const BEHIND_RELAY = [1, 2]; +// A node names itself in its messages by its socket address, a stub by its IP. +const bare = (address) => String(address).replace(/:16127$/, ''); function appComponent(appName) { return { @@ -54,10 +57,25 @@ describe('an announcement is judged one entry at a time', function () { dumpLogsOnFailure(() => env); let stub; const stamp = Date.now(); - const stubAddress = () => socketAddr(STUB_INDEX + 1); + const stubIp = () => getSubnetConfig().nodeIp(STUB_INDEX + 1); const rowNamesAt = async (i, address) => (await dbClient(i + 1).getAppLocationsByIp(address)).map((row) => row.name); - const everyNodeHolds = (address, name) => async () => (await Promise.all(realNodes.map((i) => rowNamesAt(i, address)))) - .every((names) => names.includes(name)); + const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); + // Every node stored a message from `ip` naming `name`, after `afterIds`. + const everyNodeStored = (afterIds, ip, name, timeout = 60000) => Promise.all(realNodes.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(ip) && d.apps?.some((a) => a.name === name), + timeout, + { afterId: afterIds[i] }, + ))); + const refusedOn = (i, afterId, ip, name, timeout = 60000) => env.clients[i].waitForEvent( + 'network:apprunningRefused', + (d) => d.path === 'gossip' && d.refusals?.some((r) => bare(r.ip) === bare(ip) && r.names.includes(name)), + timeout, + { afterId }, + ); + const message = (apps) => ({ + type: 'fluxapprunning', version: 2, apps, ip: stubIp(), broadcastedAt: Date.now(), osUptime: 10000, staticIp: false, + }); before(async function () { this.timeout(420000); @@ -71,9 +89,20 @@ describe('an announcement is judged one entry at a time', function () { }); await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); stub = env.stubPeerClients.get(STUB_INDEX); - await waitFor(async () => (await stub.connectedNodes()) >= 1, { - timeout: 120000, interval: 2000, label: 'node 0 is connected to the stub peer', - }); + + // Nodes 1 and 2 cannot reach the stub, whatever their own peering would choose. + await Promise.all(BEHIND_RELAY.map(async (i) => { + const cut = await execInContainer(env.clients[i].container, + `iptables -I INPUT -s ${stubIp()} -j DROP && iptables -I OUTPUT -d ${stubIp()} -j DROP`); + if (cut.exitCode !== 0) throw new Error(`fixture: node ${i} could not cut the stub off (exit ${cut.exitCode}): ${cut.output}`); + })); + await waitFor(async () => { + const peered = await Promise.all(BEHIND_RELAY.map(async (i) => { + const [outgoing, incoming] = await Promise.all([env.clients[i].getPeers(), env.clients[i].getIncomingPeers()]); + return JSON.stringify([...(outgoing.data ?? []), ...(incoming.data ?? [])]).includes(stubIp()); + })); + return peered.every((p) => !p) && (await stub.connectedNodes()) === 1; + }, { timeout: 180000, interval: 3000, label: 'only node 0 is peered with the stub' }); }); after(async function () { @@ -81,82 +110,57 @@ describe('an announcement is judged one entry at a time', function () { await env?.teardown(); }); - // Nodes 1 and 2 must hear the stub only through node 0, or the relay is not - // what carried the message to them. - async function assertOnlyNodeZeroHearsTheStub() { - const stubIp = stubAddress().split(':')[0]; - // eslint-disable-next-line no-restricted-syntax - for (const i of [1, 2]) { - // eslint-disable-next-line no-await-in-loop - const [outgoing, incoming] = await Promise.all([env.clients[i].getPeers(), env.clients[i].getIncomingPeers()]); - const peers = JSON.stringify([...(outgoing.data ?? []), ...(incoming.data ?? [])]); - expect(peers.includes(stubIp), `fixture: node ${i} is peered with the stub directly`).to.equal(false); - } - } - it('stores and relays the entries with a name and hash, and refuses the one without', async function () { this.timeout(120000); - await assertOnlyNodeZeroHearsTheStub(); const kept = `e2eentrykept${stamp}`; const refused = `e2eentryrefused${stamp}`; + const afterIds = marks(); - const sent = await stub.broadcast({ - type: 'fluxapprunning', - version: 2, - apps: [ - { name: kept, hash: `hash-${kept}`, runningSince: new Date().toISOString() }, - { name: refused, runningSince: new Date().toISOString() }, - ], - ip: stubAddress(), - broadcastedAt: Date.now(), - osUptime: 10000, - staticIp: false, - }); - expect(sent.sent, 'fixture: the message reached node 0').to.equal(1); + const sent = await stub.broadcast(message([ + { name: kept, hash: `hash-${kept}`, runningSince: new Date().toISOString() }, + { name: refused, runningSince: new Date().toISOString() }, + ])); + expect(sent.sent, 'fixture: the message reached node 0 and no other').to.equal(1); - await waitFor(everyNodeHolds(stubAddress(), kept), { - timeout: 60000, interval: 2000, label: 'every node, the two behind node 0 included, holds the entry with a hash', + const stored = await everyNodeStored(afterIds, stubIp(), kept); + stored.forEach(({ data }, i) => { + expect(data.apps.map((a) => a.name), `node ${i} stored`).to.include(kept); }); + await Promise.all(realNodes.map((i) => refusedOn(i, afterIds[i], stubIp(), refused))); // eslint-disable-next-line no-restricted-syntax for (const i of realNodes) { // eslint-disable-next-line no-await-in-loop - expect(await rowNamesAt(i, stubAddress()), `node ${i} stored the entry without a hash`).to.not.include(refused); + expect(await rowNamesAt(i, stubIp()), `node ${i} holds a row for the entry without a hash`).to.not.include(refused); } }); // An empty v2 message deletes the sender's rows. A message whose entries were - // all refused named apps, and is never read as one that named none. - it('keeps the sender\'s rows when every entry of a message is refused', async function () { - this.timeout(120000); + // all refused named apps: it stores nothing, goes no further, and is never + // read as one that named none. + it('stores and relays nothing of a message whose every entry is refused, and keeps the sender\'s rows', async function () { + this.timeout(180000); const held = `e2eentryheld${stamp}`; + const refusedOnly = `e2eentryrefusedonly${stamp}`; const marker = `e2eentrymarker${stamp}`; - const heldAt = Date.now(); - await stub.runApp(held, { hash: `hash-${held}`, broadcastedAt: heldAt }); - await waitFor(everyNodeHolds(stubAddress(), held), { - timeout: 60000, interval: 2000, label: 'every node holds the stub peer\'s app', - }); + let afterIds = marks(); + await stub.runApp(held, { hash: `hash-${held}` }); + await everyNodeStored(afterIds, stubIp(), held); - await stub.broadcast({ - type: 'fluxapprunning', - version: 2, - apps: [{ name: `e2eentryrefusedonly${stamp}` }], - ip: stubAddress(), - broadcastedAt: Date.now(), - osUptime: 10000, - staticIp: false, - }); - // A node handles one peer's messages in the order they were sent, so node 0 - // holding the marker has handled the refused message before it. + afterIds = marks(); + await stub.broadcast(message([{ name: refusedOnly }])); + // A node handles one peer's messages in the order they were sent, so a node + // that stored this has handled the refused message before it. await stub.runApp(marker, { hash: `hash-${marker}` }); - await waitFor(everyNodeHolds(stubAddress(), marker), { - timeout: 60000, interval: 2000, label: 'every node holds the message sent after the refused one', - }); + await everyNodeStored(afterIds, stubIp(), marker); - expect(Date.now() - heldAt, 'fixture: the app\'s rows have not reached their own expiry').to.be.below(50000); + await refusedOn(0, afterIds[0], stubIp(), refusedOnly); + // Relayed, it would have reached them before the marker did. + await Promise.all(BEHIND_RELAY.map((i) => assertNoEvent(env.clients[i], 'network:apprunningRefused', + (d) => d.refusals?.some((r) => r.names.includes(refusedOnly)), 0, { afterId: afterIds[i] }))); // eslint-disable-next-line no-restricted-syntax for (const i of realNodes) { // eslint-disable-next-line no-await-in-loop - expect(await rowNamesAt(i, stubAddress()), `node ${i} deleted the sender's rows for a message whose entries it refused`).to.include(held); + expect(await rowNamesAt(i, stubIp()), `node ${i} deleted the sender's rows`).to.include(held); } }); @@ -165,27 +169,23 @@ describe('an announcement is judged one entry at a time', function () { it('deletes the sender\'s rows on a message that names no apps', async function () { this.timeout(120000); const held = `e2eentrycleared${stamp}`; + let afterIds = marks(); await stub.runApp(held, { hash: `hash-${held}` }); - await waitFor(everyNodeHolds(stubAddress(), held), { - timeout: 60000, interval: 2000, label: 'every node holds the stub peer\'s app', - }); + await everyNodeStored(afterIds, stubIp(), held); - await stub.broadcast({ - type: 'fluxapprunning', - version: 2, - apps: [], - ip: stubAddress(), - broadcastedAt: Date.now(), - osUptime: 10000, - staticIp: false, - }); - - await waitFor(async () => (await Promise.all(realNodes.map((i) => rowNamesAt(i, stubAddress())))).every((names) => names.length === 0), { - timeout: 60000, interval: 2000, label: 'every node dropped the stub peer\'s rows', - }); + afterIds = marks(); + await stub.broadcast(message([])); + await Promise.all(realNodes.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', (d) => bare(d.ip) === bare(stubIp()) && d.apps?.length === 0, 60000, { afterId: afterIds[i] }, + ))); + // eslint-disable-next-line no-restricted-syntax + for (const i of realNodes) { + // eslint-disable-next-line no-await-in-loop + expect(await rowNamesAt(i, stubIp()), `node ${i} kept the sender's rows`).to.deep.equal([]); + } }); - it('keeps a node\'s apps standing while it holds an app installed without a hash', async function () { + it('leaves out an app installed without a hash, and every node goes on storing the rest', async function () { this.timeout(420000); const appName = `e2eentryreal${stamp}`; await pushImage(appName, 'v1'); @@ -193,37 +193,27 @@ describe('an announcement is judged one entry at a time', function () { const idx = await seedAndInstall(env, app); const holder = socketAddr(idx + 1); const others = realNodes.filter((i) => i !== idx); - await waitFor(everyNodeHolds(holder, appName), { - timeout: 180000, interval: 3000, label: 'every node holds the app at its holder', - }); // An app installed without a hash, as a node carries one it took before it // upgraded. Everything else about it is the real app's. const unhashed = `e2eentryunhashed${stamp}`; - const installed = await dbClient(idx + 1).localApp(appName); - const { hash, ...withoutHash } = installed; + const { hash, ...withoutHash } = await dbClient(idx + 1).localApp(appName); expect(hash, 'fixture: the real app is installed with a hash').to.be.a('string'); - const seededAt = Date.now(); + const afterIds = marks(); await dbClient(idx + 1).seedLocalApp({ ...withoutHash, name: unhashed }); - const mark = env.clients[idx].getLastEventId(); - const announced = await env.clients[idx].waitForEvent('app:running', (d) => d.apps?.some((a) => a.name === appName), 120000, { afterId: mark }); - expect(announced.apps.map((a) => a.name), 'the holder announced the app it has no hash for').to.not.include(unhashed); + const { data: announced } = await env.clients[idx].waitForEvent( + 'app:running', (d) => d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[idx] }, + ); expect(await dbClient(idx + 1).localApp(unhashed), 'fixture: the unhashed app was still installed when the holder announced').to.not.equal(null); - - // Two location lifetimes after the seeding, the app's rows on every other node - // are ones written since: a holder whose announcements were refused would have - // let them lapse. - await waitFor(async () => Date.now() - seededAt > 130000, { timeout: 140000, interval: 5000, label: 'two location lifetimes pass' }); - // eslint-disable-next-line no-restricted-syntax - for (const i of others) { - // eslint-disable-next-line no-await-in-loop - const row = await dbClient(i + 1).appLocationAt(appName, holder); - expect(row, `node ${i} lost the holder's row`).to.not.equal(null); - expect(new Date(row.broadcastedAt).getTime(), `node ${i} holds no announcement made since`).to.be.above(seededAt); - } - const own = await dbClient(idx + 1).appLocationAt(appName, holder); - expect(own, 'the holder lost its own row').to.not.equal(null); - expect(new Date(own.broadcastedAt).getTime(), 'the holder stored none of its own announcements').to.be.above(seededAt); + expect(announced.apps.map((a) => a.name), 'the holder announced the app it has no hash for').to.not.include(unhashed); + expect(announced.unannounced, 'the holder did not say what it left out').to.deep.equal([unhashed]); + + // Every other node stores the announcements made since. + await Promise.all(others.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[i] }, + ).catch(() => { throw new Error(`node ${i} stored no announcement from the holder since the unhashed app was installed`); }))); + await Promise.all(others.map((i) => assertNoEvent(env.clients[i], 'network:apprunningRefused', + (d) => d.refusals?.some((r) => bare(r.ip) === bare(holder)), 0, { afterId: afterIds[i] }))); }); }); diff --git a/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js index fbb81217da..1199740cd0 100644 --- a/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js +++ b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js @@ -7,7 +7,7 @@ import { createTestEnv } from '../framework/test-env.js'; import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; -import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { waitForBootSettled, assertNoEvent } from '../framework/wait.js'; import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; import { restartFluxos } from '../framework/container.js'; @@ -16,16 +16,19 @@ import { fluxTeamKey } from '../framework/keys.js'; import { authenticate } from '../auth.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; -// A node records when it took an app, once, and announces that as the app's -// runningSince for as long as the app stays on it. The network's copy of the -// node's own location row is not where it comes from: that row lapses while the -// app goes on running, and a redeploy removes and reinstalls the app without it -// leaving. The record goes with the app, and at boot the records are made to -// match what is installed, dating any missing one at the node's own row. +// A node records when it took an app, once (app:placementRecorded), and announces +// that as the app's runningSince for as long as the app stays on it. The +// network's copy of the node's own location row is not where it comes from: that +// row lapses while the app goes on running, and a redeploy removes and +// reinstalls the app without it leaving. The record goes with the app +// (app:placementRemoved), and at boot the records are made to match what is +// installed (app:placementsReconciled), dating a missing one at the node's own +// row. const REAL_NODES = 3; const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); const DAY_MS = 24 * 60 * 60 * 1000; +const bare = (address) => String(address).replace(/:16127$/, ''); describe('a node keeps its own record of when it took an app', function () { let env; @@ -33,24 +36,26 @@ describe('a node keeps its own record of when it took an app', function () { let idx; let holder; let others; - let placedAt; + // What the holder's announcements carry for the app, from the first one on. + let announcedSince; const appName = `e2eplacement${Date.now()}`; - const placementOf = async (name) => (await dbClient(idx + 1).placements())[name]; - // The runningSince every other node holds for the holder, once each has stored - // an announcement made after `since`. - async function announcedSince(since) { - const rows = await Promise.all(others.map((i) => dbClient(i + 1).appLocationAt(appName, holder))); - if (!rows.every((row) => row && new Date(row.broadcastedAt).getTime() > since)) return null; - return rows.map((row) => new Date(row.runningSince).getTime()); - } - async function waitForAnnouncedSince(since, label) { - let seen = null; - await waitFor(async () => { seen = await announcedSince(since); return seen !== null; }, { - timeout: 120000, interval: 3000, label, - }); - return seen; + const runningSinceIn = (data) => { + const entry = data.apps?.find((a) => a.name === appName); + return entry ? new Date(entry.runningSince).getTime() : null; + }; + // The runningSince every other node stored from the holder's next announcement + // after `afterIds`. + async function nextStoredRunningSince(afterIds) { + const events = await Promise.all(others.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), + 120000, + { afterId: afterIds[i] }, + ))); + return events.map(({ data }) => runningSinceIn(data)); } + const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); const teamAuth = async () => (await authenticate(env.clients[idx].url, fluxTeamKey())).zelidauth; before(async function () { @@ -80,6 +85,8 @@ describe('a node keeps its own record of when it took an app', function () { idx = await seedAndInstall(env, app); holder = socketAddr(idx + 1); others = realNodes.filter((i) => i !== idx); + const { data: first } = await env.clients[idx].waitForEvent('app:running', (d) => runningSinceIn(d) !== null, 120000); + announcedSince = runningSinceIn(first); }); after(async function () { @@ -87,39 +94,38 @@ describe('a node keeps its own record of when it took an app', function () { await env?.teardown(); }); - it('records when it took the app, and announces that', async function () { + it('records when it took the app at install, and announces that', async function () { this.timeout(180000); - const recorded = await placementOf(appName); - expect(recorded, 'no record of when the holder took the app').to.be.instanceOf(Date); - placedAt = recorded.getTime(); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000); + expect(recorded.source).to.equal('install'); + expect(new Date(recorded.placedAt).getTime(), 'the announced runningSince is not the recorded one').to.equal(announcedSince); - const seen = await waitForAnnouncedSince(placedAt, 'every other node holds an announcement made since the install'); - expect(seen, 'the announced runningSince is not the recorded one').to.deep.equal(others.map(() => placedAt)); + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'another node stored a different runningSince').to.deep.equal(others.map(() => announcedSince)); }); - it('keeps announcing the recorded time after its own location rows lapse', async function () { + it('keeps announcing the same time after its own location rows lapse', async function () { this.timeout(180000); + const afterIds = marks(); // Every node's row for the holder, the holder's own included, as after an // expiry, an eviction or a sigterm. await Promise.all(realNodes.map((i) => dbClient(i + 1).deleteAppLocation(appName, holder))); - const lapsedAt = Date.now(); - const seen = await waitForAnnouncedSince(lapsedAt, 'every other node holds an announcement made since the rows lapsed'); - expect(seen, 'the holder re-dated the app when its row lapsed').to.deep.equal(others.map(() => placedAt)); - expect((await placementOf(appName)).getTime(), 'the record moved').to.equal(placedAt); + const stored = await nextStoredRunningSince(afterIds); + expect(stored, 'the holder re-dated the app when its row lapsed').to.deep.equal(others.map(() => announcedSince)); }); - it('keeps the record through a hard redeploy', async function () { + it('keeps the record, and the time it announces, through a hard redeploy', async function () { this.timeout(300000); - const mark = env.clients[idx].getLastEventId(); + const afterIds = marks(); const body = await env.clients[idx].redeployApp(appName, await teamAuth(), { force: true }); - const rebuilt = await env.clients[idx].waitForEvent('app:rebuilt', (d) => d.name === appName, 240000, { afterId: mark }); + const { data: rebuilt } = await env.clients[idx].waitForEvent('app:rebuilt', (d) => d.name === appName, 240000, { afterId: afterIds[idx] }); expect(rebuilt.outcome, `the redeploy did not reinstall the app: ${body.slice(-400)}`).to.equal('installed'); - const redeployedAt = Date.now(); - expect((await placementOf(appName)).getTime(), 'the redeploy moved the record').to.equal(placedAt); - const seen = await waitForAnnouncedSince(redeployedAt, 'every other node holds an announcement made since the redeploy'); - expect(seen, 'the redeploy re-dated the app').to.deep.equal(others.map(() => placedAt)); + await assertNoEvent(env.clients[idx], 'app:placementRemoved', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'the redeploy re-dated the app').to.deep.equal(others.map(() => announcedSince)); }); // A node that took an app before it had these records gets one at boot, dated @@ -134,29 +140,27 @@ describe('a node keeps its own record of when it took an app', function () { await dbClient(idx + 1).setAppLocationRunningSince(appName, holder, rowSince); // FluxOS alone, so the downtime is seconds and the node keeps its apps. - const mark = env.clients[idx].getLastEventId(); + const afterIds = marks(); await restartFluxos(env.clients[idx].container); - await waitForBootSettled(env.clients[idx], 240000, { afterId: mark }); - - const records = await dbClient(idx + 1).placements(); - expect(records[appName]?.getTime(), 'the missing record was not dated at the node\'s own row').to.equal(rowSince); - expect(records, 'the record of an app the node does not hold survived boot').to.not.have.property(ghost); - const bootedAt = Date.now(); - const seen = await waitForAnnouncedSince(bootedAt, 'every other node holds an announcement made since the boot'); - expect(seen, 'the announced runningSince is not the recorded one').to.deep.equal(others.map(() => rowSince)); - placedAt = rowSince; + await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); + + const { data: reconciled } = await env.clients[idx].waitForEvent('app:placementsReconciled', () => true, 10000, { afterId: afterIds[idx] }); + expect(reconciled, 'the boot did not reconcile the records').to.deep.equal({ created: 1, fromLocationRows: 1, removed: 1 }); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); + expect(recorded.source).to.equal('locationRow'); + expect(new Date(recorded.placedAt).getTime(), 'the missing record was not dated at the node\'s own row').to.equal(rowSince); + + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'the announced runningSince is not the recorded one').to.deep.equal(others.map(() => rowSince)); }); - it('drops the record with the app, so a later install starts a new one', async function () { + it('drops the record with the app', async function () { this.timeout(240000); - const mark = env.clients[idx].getLastEventId(); + const afterIds = marks(); const body = await env.clients[idx].removeAppLocally(appName, await teamAuth()); - await env.clients[idx].waitForEvent('app:removed', (d) => d.name === appName, 180000, { afterId: mark }); + await env.clients[idx].waitForEvent('app:removed', (d) => d.name === appName, 180000, { afterId: afterIds[idx] }) + .catch(() => { throw new Error(`the app was not removed: ${body.slice(-400)}`); }); - // The node's own spawner may take the app back at once; whichever record - // stands is a new one, never the one the removal ended. - const after = await placementOf(appName); - expect(after?.getTime(), `the removal kept the record: ${body.slice(-400)}`).to.not.equal(placedAt); - if (after) expect(after.getTime(), 'a record older than the removal').to.be.at.least(Date.now() - 180000); + await env.clients[idx].waitForEvent('app:placementRemoved', (d) => d.name === appName, 30000, { afterId: afterIds[idx] }); }); }); diff --git a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js index 9a494cac7e..5e782c86cb 100644 --- a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js +++ b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js @@ -9,10 +9,9 @@ import { appOwnerKey, fluxTeamKey, nodeKey, userKey, } from '../framework/keys.js'; import { authenticate } from '../auth.js'; -import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { waitFor, waitForBootSettled, assertNoEvent } from '../framework/wait.js'; import { bootAndPeer } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; -import { socketAddr } from '../framework/state-events.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // An app registered and not yet paid for exists only as a temporary message. @@ -32,12 +31,15 @@ describe('who may place or test an app by hand', function () { const client = () => env.clients[NODE]; const authAs = async (key) => (await authenticate(client().url, key)).zelidauth; - // Nothing of the app is left on the node, and no node holds a location for it there. - async function assertNothingHeld() { - expect(await dbClient(NODE + 1).localApp(appName), 'the app is installed on the node').to.equal(null); - expect(await dbClient(NODE + 1).placements(), 'the node holds a placement record for the app').to.not.have.property(appName); - const rows = await Promise.all(env.clients.map((_c, i) => dbClient(i + 1).appLocationAt(appName, socketAddr(NODE + 1)))); - expect(rows.filter(Boolean), 'a node holds a location for the app at the node').to.deep.equal([]); + const marks = () => env.clients.map((c) => c.getLastEventId()); + // Since `afterIds` the node recorded no placement of the app and announced it + // nowhere, and nothing of it is left installed. + async function assertNothingHeld(afterIds) { + await assertNoEvent(client(), 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[NODE] }); + await assertNoEvent(client(), 'app:running', (d) => d.apps?.some((a) => a.name === appName), 0, { afterId: afterIds[NODE] }); + await Promise.all(env.clients.map((c, i) => assertNoEvent(c, 'network:apprunning', + (d) => d.apps?.some((a) => a.name === appName), 0, { afterId: afterIds[i] }))); + expect(await dbClient(NODE + 1).localApp(appName), 'the app is left installed on the node').to.equal(null); } before(async function () { @@ -69,47 +71,52 @@ describe('who may place or test an app by hand', function () { it('refuses its owner placing it with installapplocally', async function () { this.timeout(120000); + const afterIds = marks(); const body = await client().installAppLocally(messageHash, await authAs(appOwnerKey())); expect(body, 'the owner was not refused').to.match(/Unauthorized/); - await assertNothingHeld(); + await assertNothingHeld(afterIds); }); it('does not place it for the flux team either', async function () { this.timeout(120000); + const afterIds = marks(); const body = await client().installAppLocally(messageHash, await authAs(fluxTeamKey())); expect(body, 'installapplocally read the temporary message').to.match(new RegExp(`Application Specifications of ${messageHash} not found`)); - await assertNothingHeld(); + await assertNothingHeld(afterIds); }); it('refuses another user testing it', async function () { this.timeout(120000); + const afterIds = marks(); const body = await client().testAppInstall(messageHash, await authAs(userKey())); expect(body, 'another user was not refused').to.match(/Unauthorized/); - await assertNothingHeld(); + await assertNothingHeld(afterIds); }); it('lets its owner test it, and leaves nothing behind', async function () { this.timeout(300000); + const afterIds = marks(); const body = await client().testAppInstall(messageHash, await authAs(appOwnerKey())); expect(body, 'the owner\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); await waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { timeout: 60000, interval: 2000, label: 'the test install is torn down', }); - await assertNothingHeld(); + await assertNothingHeld(afterIds); }); it('lets the flux team test it', async function () { this.timeout(300000); + const afterIds = marks(); const body = await client().testAppInstall(messageHash, await authAs(fluxTeamKey())); expect(body, 'the flux team\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); await waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { timeout: 60000, interval: 2000, label: 'the test install is torn down', }); - await assertNothingHeld(); + await assertNothingHeld(afterIds); }); }); From 2520bae1aafdf5c597629c3e76946d777cda32f3 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:16:39 +0100 Subject: [PATCH 334/369] test(harness): 3501 cuts the stub off before the fleet peers; 3503 waits for a peer each way Co-Authored-By: Claude Opus 5.5 --- ...ouncement-is-judged-one-entry-at-a-time.js | 21 +++++++++---------- ...03-who-may-place-or-test-an-app-by-hand.js | 8 +++++-- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js index 6d68a2767a..b50ea9f3c5 100644 --- a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js +++ b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js @@ -7,7 +7,7 @@ import { createTestEnv } from '../framework/test-env.js'; import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; -import { waitFor, assertNoEvent } from '../framework/wait.js'; +import { assertNoEvent } from '../framework/wait.js'; import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; import { execInContainer } from '../framework/container.js'; @@ -87,22 +87,21 @@ describe('an announcement is judged one entry at a time', function () { tickerAutostart: false, configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, }); - await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); stub = env.stubPeerClients.get(STUB_INDEX); - - // Nodes 1 and 2 cannot reach the stub, whatever their own peering would choose. + // Nodes 1 and 2 cannot reach the stub, cut off before the fleet peers so + // neither ever holds a connection to it. await Promise.all(BEHIND_RELAY.map(async (i) => { const cut = await execInContainer(env.clients[i].container, `iptables -I INPUT -s ${stubIp()} -j DROP && iptables -I OUTPUT -d ${stubIp()} -j DROP`); if (cut.exitCode !== 0) throw new Error(`fixture: node ${i} could not cut the stub off (exit ${cut.exitCode}): ${cut.output}`); })); - await waitFor(async () => { - const peered = await Promise.all(BEHIND_RELAY.map(async (i) => { - const [outgoing, incoming] = await Promise.all([env.clients[i].getPeers(), env.clients[i].getIncomingPeers()]); - return JSON.stringify([...(outgoing.data ?? []), ...(incoming.data ?? [])]).includes(stubIp()); - })); - return peered.every((p) => !p) && (await stub.connectedNodes()) === 1; - }, { timeout: 180000, interval: 3000, label: 'only node 0 is peered with the stub' }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + const peeredWithStub = await Promise.all(BEHIND_RELAY.map(async (i) => { + const [outgoing, incoming] = await Promise.all([env.clients[i].getPeers(), env.clients[i].getIncomingPeers()]); + return JSON.stringify([...(outgoing.data ?? []), ...(incoming.data ?? [])]).includes(stubIp()); + })); + expect(peeredWithStub, 'fixture: a node behind the relay is peered with the stub').to.deep.equal([false, false]); + expect(await stub.connectedNodes(), 'fixture: the stub holds connections beyond node 0').to.equal(1); }); after(async function () { diff --git a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js index 5e782c86cb..97aade058b 100644 --- a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js +++ b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js @@ -20,7 +20,7 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // down, and never announces it - for its owner or the flux team, and refuses any // other caller before a container is made. -const REAL_NODES = 2; +const REAL_NODES = 3; const NODE = 0; describe('who may place or test an app by hand', function () { @@ -44,7 +44,7 @@ describe('who may place or test an app by hand', function () { before(async function () { this.timeout(420000); - // Registration wants this many peers each way, and two nodes hold one. + // Registration wants peers both ways; a fleet this small holds one each way. env = await createTestEnv({ hookCtx: this, nodes: REAL_NODES, @@ -53,6 +53,10 @@ describe('who may place or test an app by hand', function () { }); await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); await Promise.all(env.clients.map((c) => waitForBootSettled(c, 240000))); + await waitFor(async () => { + const [outgoing, incoming] = await Promise.all([client().getPeers(), client().getIncomingPeers()]); + return (outgoing.data?.length ?? 0) >= 1 && (incoming.data?.length ?? 0) >= 1; + }, { timeout: 120000, interval: 2000, label: 'the registering node has a peer each way' }); appName = `e2ehandplaced${Date.now()}`; const spec = buildAppSpec({ name: appName }); From 639495bf9c4e3006b3ac2721c640774b23ce3566 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:24:22 +0100 Subject: [PATCH 335/369] test(harness): 3501 places the unhashed-app case on a node that announces on its own interval Co-Authored-By: Claude Opus 5.5 --- .../3501-an-announcement-is-judged-one-entry-at-a-time.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js index b50ea9f3c5..4a3cc47e3e 100644 --- a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js +++ b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js @@ -8,7 +8,7 @@ import { pushImage } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; import { assertNoEvent } from '../framework/wait.js'; -import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; import { execInContainer } from '../framework/container.js'; import { socketAddr } from '../framework/state-events.js'; @@ -189,7 +189,11 @@ describe('an announcement is judged one entry at a time', function () { const appName = `e2eentryreal${stamp}`; await pushImage(appName, 'v1'); const app = await buildSeedableApp({ name: appName, instances: 1, compose: [appComponent(appName)] }); - const idx = await seedAndInstall(env, app); + // Node 1: it announces on its own interval. Node 2's ring neighbour is the + // stub's slot, which it cannot reach, so it holds fewer peers than the + // threshold that starts a node's announcements. + const idx = 1; + await installOnNodes(env, app, [idx]); const holder = socketAddr(idx + 1); const others = realNodes.filter((i) => i !== idx); From afc59017a2ea91fe469209db606e036f26046380 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:39:43 +0100 Subject: [PATCH 336/369] fix(apps): installapplocally refuses a specification without a height An app's term runs from the height its payment was mined at, and the expiry sweep removes the app when the term ends. installapplocally installs a specification only with a height; a height of zero, a permanent local app, is one. ownerAppAllowance and temporaryAppAllowance have no reader left and are removed from the configuration. Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 2 - .../src/services/appLifecycle/appInstaller.js | 26 ++++--------- tests/unit/appInstaller.test.js | 38 ++++++++++++++++++- tests/unit/globalconfig/default.js | 2 - 4 files changed, 43 insertions(+), 25 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index d4a1e63590..42d408dff4 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -595,8 +595,6 @@ module.exports = { daemonPONFork: 2020000, // block height where PON (Proof of Node) fork activates - chain works 4x faster after this block blocksAllowanceInterval: 1000, // ap differences can be in 1000s - more than 1 day removeBlocksAllowanceIntervalBlock: 1625000, // after this block we can start having app updates without extending subscription - block expected in April 19th 2024 - ownerAppAllowance: 1000, // a by-name local install (fluxteam only) runs for this amount of blocks before the expiry sweep removes it - temporaryAppAllowance: 200, // in case of any user installing some temporary app message for testing purposes, the app will run for this many blocks expireFluxAppsPeriod: 100, // every 100 blocks we run a check that deletes apps specifications and stops/removes the application from existence if it has been lastly updated more than 22k blocks ago updateFluxAppsPeriod: 9, // every 9 blocks we check for reinstalling of old application versions removeFluxAppsPeriod: 11, // every 11 blocks we check for more than maximum number of instances of an application diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 8f9bd5b64f..47df1ea822 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -34,7 +34,7 @@ const mountParser = require('../utils/mountParser'); const { ensureStignoreCovers } = require('../appSystem/syncthingIgnorePolicy'); const { findCommonArchitectures } = require('../utils/appUtilities'); const log = require('../../lib/log'); -const { localAppsInformation, scannedHeightCollection } = require('../utils/appConstants'); +const { localAppsInformation } = require('../utils/appConstants'); const appPlacements = require('../appManagement/appPlacements'); const messageVerifier = require('../appMessaging/messageVerifier'); const { availableApps, getApplicationGlobalSpecifications } = require('../appDatabase/registryManager'); @@ -1089,25 +1089,13 @@ async function installAppLocally(req, res) { appSpecifications.hash = hash; } - // get current height - const dbopen = dbHelper.databaseConnection(); - if (!appSpecifications.height && appSpecifications.height !== 0) { - // A specification without a height runs for ownerAppAllowance blocks from now. - const database = dbopen.db(config.database.daemon.database); - const query = { generalScannedHeight: { $gte: 0 } }; - const projection = { - projection: { - _id: 0, - generalScannedHeight: 1, - }, - }; - const result = await dbHelper.findOneInDatabase(database, scannedHeightCollection, query, projection); - if (!result) { - throw new Error('Scanning not initiated'); - } - const explorerHeight = serviceHelper.ensureNumber(result.generalScannedHeight); - appSpecifications.height = explorerHeight - config.fluxapps.blocksLasting + config.fluxapps.ownerAppAllowance; + // An app's term runs from the height its payment was mined at, and the + // expiry sweep removes it when the term ends. A specification without one + // has no term to run, so it is not installed. + if (appSpecifications.height === undefined || appSpecifications.height === null) { + throw new Error(`Application Specifications of ${appname} carry no height`); } + const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); const appsQuery = {}; // all diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 74ec310ef2..80602d23c3 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -87,8 +87,6 @@ describe('appInstaller tests', () => { fluxapps: { blocksLasting: 22000, latestAppSpecification: 1, - ownerAppAllowance: 100, - temporaryAppAllowance: 200, maxImageSize: 10000000000, }, }; @@ -525,6 +523,42 @@ describe('appInstaller tests', () => { expect(hwRequirementsStub.checkAppHWRequirements.called, 'nothing is installed').to.be.false; }); + // An app's term runs from the height its payment was mined at; one without a + // height has no term, and nothing would ever expire it. + it('refuses a specification that carries no height', async () => { + const req = { params: { appname: 'noheightapp' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + baseStubs['../appDatabase/registryManager'].availableApps.resolves([{ name: 'noheightapp', version: 3, owner: 'someone' }]); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(res.json.calledOnce).to.be.true; + expect(res.json.firstCall.args[0].data.message).to.equal('Application Specifications of noheightapp carry no height'); + expect(hwRequirementsStub.checkAppHWRequirements.called, 'nothing is installed').to.be.false; + expect(dbHelperStub.insertOneToDatabase.called, 'nothing is installed').to.be.false; + }); + + it('installs a permanent-lasting local app, whose height is zero', async () => { + const req = { params: { appname: 'foreverapp' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + baseStubs['../appDatabase/registryManager'].availableApps.resolves([{ name: 'foreverapp', version: 3, owner: 'someone', height: 0 }]); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + hwRequirementsStub.checkAppHWRequirements.rejects(new Error('stop here')); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(hwRequirementsStub.checkAppHWRequirements.calledOnce, 'a height of zero was refused').to.be.true; + }); + it('installs a permanent message with the hash and height the message carries', async () => { const req = { params: { appname: 'permhash' }, query: {} }; const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index a9127a31bc..561c4890f4 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -302,8 +302,6 @@ module.exports = { daemonPONFork: 2020000, // block height where PON (Proof of Node) fork activates - chain works 4x faster after this block blocksAllowanceInterval: 1000, // ap differences can be in 1000s - more than 1 day removeBlocksAllowanceIntervalBlock: 1625000, // after this block we can start having app updates without extending subscription - block expected in April 19th 2024 - ownerAppAllowance: 1000, // in case of node owner installing some app, the app will run for this amount of blocks - temporaryAppAllowance: 200, // in case of any user installing some temporary app message for testing purposes, the app will run for this many blocks expireFluxAppsPeriod: 100, // every 100 blocks we run a check that deletes apps specifications and stops/removes the application from existence if it has been lastly updated more than 22k blocks ago updateFluxAppsPeriod: 9, // every 9 blocks we check for reinstalling of old application versions removeFluxAppsPeriod: 11, // every 11 blocks we check for more than maximum number of instances of an application From fcde09d005a8e6e7a72d37d26cfeb5e3979f331b Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:40:35 +0100 Subject: [PATCH 337/369] test(apps): a redeploy for a change of components keeps the placement record, and drops it when the app does not come back Co-Authored-By: Claude Opus 5.5 --- tests/unit/advancedWorkflows.test.js | 42 ++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 6c3abf18aa..43632d6087 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -27,6 +27,7 @@ const { PeerComponent } = require('../../ZelBack/src/services/appMonitoring/peer const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../../ZelBack/src/services/utils/addressMoveEvents'); const appInstaller = require('../../ZelBack/src/services/appLifecycle/appInstaller'); +const appPlacements = require('../../ZelBack/src/services/appManagement/appPlacements'); const appRebuild = require('../../ZelBack/src/services/appLifecycle/appRebuild'); const appNetworkLinker = require('../../ZelBack/src/services/appLifecycle/appNetworkLinker'); const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); @@ -7552,6 +7553,47 @@ describe('advancedWorkflows tests', () => { ).to.be.false; }); + // A version 8 app whose components change is removed and installed again on + // this node, so the record of when the node took it is kept - unless the + // reinstall does not bring the app back. + describe('a redeploy for a change of components', () => { + const v8Installed = { ...installedApp, version: 8, enterprise: '' }; + const v8New = { + ...v8Installed, + hash: 'newhash', + compose: [...v8Installed.compose, { ...v8Installed.compose[0], name: 'db', ports: ['31001'] }], + }; + let removeAppLocally; + let registerAppLocally; + let removePlacement; + + beforeEach(() => { + dbHelper.findInDatabase.resolves([v8Installed]); + dbHelper.findOneInDatabase.resolves(v8New); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + registerAppLocally = sinon.stub(appInstaller, 'registerAppLocally').resolves(InstallOutcome.INSTALLED); + removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); + }); + + it('removes the app as one being redeployed, and keeps its record', async () => { + await advancedWorkflows.reinstallOldApplications(); + + expect(removeAppLocally.calledOnce, 'the fixture never reached the redeploy it is about').to.equal(true); + expect(removeAppLocally.firstCall.args[5]).to.deep.equal({ redeploying: true }); + sinon.assert.calledOnce(registerAppLocally); + sinon.assert.notCalled(removePlacement); + }); + + it('drops the record when the reinstall does not bring the app back', async () => { + registerAppLocally.resolves(InstallOutcome.FAILED); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnceWithExactly(removePlacement, 'myapp'); + }); + }); + it('does not uninstall an app whose redeploy stops on a database that does not answer', async () => { sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); From f0950e4f89b0c646db9d498eff854bb625d0b8a0 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 08:43:03 +0100 Subject: [PATCH 338/369] test(harness): 3504 a synced announcement is judged one entry at a time; 3505 an unhashed app across a rolling upgrade - 3504: a node joining from its peers' event logs stores an announcement's readable entries and refuses the rest; a newer announcement whose every entry it refuses neither prunes the rows an older readable one stood behind nor takes its place in the joiner's own log. - 3505: with one node on the FluxOS the others upgrade from, an upgraded node holding an app installed without a hash stays visible to it, and that node holding one stays visible to the upgraded nodes, which refuse only that entry. apprunningEvent takes an entry as an object, so a refused one can be built. Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/state-events.js | 12 +- ...ouncement-is-judged-one-entry-at-a-time.js | 116 ++++++++++++++++ ...n-unhashed-app-across-a-rolling-upgrade.js | 128 ++++++++++++++++++ 3 files changed, 251 insertions(+), 5 deletions(-) create mode 100644 test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js create mode 100644 test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js diff --git a/test-infra/runner/framework/state-events.js b/test-infra/runner/framework/state-events.js index e96462a747..60aab4dc6b 100644 --- a/test-infra/runner/framework/state-events.js +++ b/test-infra/runner/framework/state-events.js @@ -22,7 +22,9 @@ async function envelopeFor(nodeNum, data, broadcastedAt, signedBy) { }; } -// A v2 apprunning broadcast: the node runs `apps`, as of broadcastedAt. +// A v2 apprunning broadcast: the node runs `apps`, as of broadcastedAt. An app +// named by a string gets a hash and a runningSince; one given as an object is +// sent as it is, so an entry a node refuses can be built. export async function apprunningEvent({ nodeNum, apps, broadcastedAt, dedupKey = 'v2', signedBy = nodeNum, }) { @@ -30,11 +32,11 @@ export async function apprunningEvent({ const data = { type: 'fluxapprunning', version: 2, - apps: apps.map((name) => ({ - name, - hash: `hash-${name}`, + apps: apps.map((app) => (typeof app === 'string' ? { + name: app, + hash: `hash-${app}`, runningSince: new Date(broadcastedAt).toISOString(), - })), + } : app)), ip, broadcastedAt, osUptime: 10000, diff --git a/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js new file mode 100644 index 0000000000..fd61c32fea --- /dev/null +++ b/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js @@ -0,0 +1,116 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { dbClient } from '../framework/db-client.js'; +import { apprunningEvent, socketAddr } from '../framework/state-events.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { waitForDaemonReady, waitForNodeStatus, waitForOrchestratorState } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node joining the network learns where apps run from its peers' event logs, +// and judges each announcement there one entry at a time, as it judges one that +// arrives by gossip: an entry without a name and a hash is refused +// (network:apprunningRefused, path sync) and the rest are stored. +// +// A peer that has not upgraded serves whatever it last recorded for a node, +// which may be an announcement whose every entry the joiner refuses. The joiner +// prunes a node's rows against the newest announcement it can store, so such a +// message never deletes the rows an older readable one stood behind, and it +// never takes that one's place in the joiner's own log. + +const RUNNING = [0, 1, 2, 3]; +const JOINER = 4; +// The nodes the announcements are about, by node number: real nodes holding no +// apps, so the seeded announcements are the only ones there are about them. +const MIXED_NODE = 3; +const REFUSED_NODE = 4; + +describe('a synced announcement is judged one entry at a time', function () { + let env; + dumpLogsOnFailure(() => env); + let joiner; + let refused; + const stamp = Date.now(); + const kept = `e2esynckept${stamp}`; + const dropped = `e2esyncdropped${stamp}`; + const held = `e2esyncheld${stamp}`; + const probe = `e2esyncprobe${stamp}`; + const rowNames = async (nodeNum) => (await dbClient(JOINER + 1).getAppLocationsByIp(socketAddr(nodeNum))).map((row) => row.name); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: RUNNING.length + 1, + deferredNodes: 1, + tickerAutostart: false, + // The joiner learns from these, and a node answers a state sync only once + // its own has finished. + syncedNodes: RUNNING, + }); + await bootAndPeer(env); + + // Booted cut off, so the seeded announcements' acceptance window spans the + // sync and not the boot. + await env.holdOutPendingNode(JOINER, RUNNING); + joiner = await env.startNode(JOINER); + await waitForDaemonReady(joiner); + await waitForNodeStatus(joiner, (d) => d.confirmed === true, 30000); + + const at = Date.now(); + const events = [ + // One readable entry and one without a hash. + await apprunningEvent({ + nodeNum: MIXED_NODE, + broadcastedAt: at, + apps: [kept, { name: dropped, runningSince: new Date(at).toISOString() }], + }), + // A readable announcement, then a newer one whose only entry has no hash: + // what a peer that has not upgraded holds for a node it last heard that from. + await apprunningEvent({ + nodeNum: REFUSED_NODE, broadcastedAt: at - 5000, apps: [held], dedupKey: 'v2:readable', + }), + await apprunningEvent({ + nodeNum: REFUSED_NODE, broadcastedAt: at, apps: [{ name: probe, runningSince: new Date(at).toISOString() }], + }), + ]; + await Promise.all(RUNNING.map((i) => dbClient(i + 1).seedAppStateEvents(events.map((event) => ({ ...event }))))); + + const mark = joiner.getLastEventId(); + await env.releasePendingNode(JOINER, RUNNING); + await env.startDiscovery([JOINER]); + await waitForOrchestratorState(joiner, 'READY', 180000); + // Every peer's response stored and pruned. + await joiner.waitForEvent('ephemeralSync:allComplete', () => true, 180000, { afterId: mark }); + refused = joiner.getEventBuffer() + .filter((e) => e.id > mark && e.event === 'network:apprunningRefused' && e.data.path === 'sync') + .flatMap((e) => e.data.refusals); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('stores the entries with a name and hash, and refuses the one without', async function () { + const names = await rowNames(MIXED_NODE); + expect(names).to.include(kept); + expect(names, 'the joiner stored an entry without a hash').to.not.include(dropped); + expect(refused.some((r) => r.ip === socketAddr(MIXED_NODE) && r.names.includes(dropped)), 'the joiner did not report the entry it refused').to.equal(true); + }); + + it('keeps a node\'s rows past a newer announcement whose every entry it refuses', async function () { + const names = await rowNames(REFUSED_NODE); + expect(names, 'the refused announcement pruned the rows the readable one stood behind').to.include(held); + expect(names, 'the joiner stored an entry without a hash').to.not.include(probe); + expect(refused.some((r) => r.ip === socketAddr(REFUSED_NODE) && r.names.includes(probe)), 'the joiner did not report the announcement it refused').to.equal(true); + }); + + it('keeps the readable announcement in the log it serves its own peers', async function () { + const logged = await dbClient(JOINER + 1).getAppStateEvents({ ip: socketAddr(REFUSED_NODE), type: 'apprunning' }); + expect(logged.map((event) => event.data.apps.map((a) => a.name)), 'the refused announcement took the readable one\'s place').to.deep.equal([[held]]); + }); +}); diff --git a/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js b/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js new file mode 100644 index 0000000000..b1af283bd5 --- /dev/null +++ b/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js @@ -0,0 +1,128 @@ +// base-image: FLUX_E2E_BASE_TAG +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A network part way through an upgrade. Node 0 runs the FluxOS the others +// upgrade from (FLUX_E2E_BASE_TAG), which refuses a whole apprunning message for +// one entry without a hash; nodes 1 and 2 run this build. +// +// An upgraded node holding an app installed without a hash leaves it out of its +// announcements (app:running.unannounced), so the node not yet upgraded goes on +// storing the rest (network:apprunning). A node not yet upgraded announces such +// an app, and the upgraded nodes store its other entries and refuse that one +// (network:apprunningRefused). +// +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming what it needs. + +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; +const NOT_UPGRADED = 0; +const UPGRADED = [1, 2]; +const bare = (address) => String(address).replace(/:16127$/, ''); + +function appComponent(appName) { + return { + name: 'alpha', + description: 'rolling upgrade', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +describe('an app installed without a hash, across a rolling upgrade', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + + // Install `appName` on `holder`, then give it an app installed without a hash. + async function holderWithUnhashedApp(appName, holder) { + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ name: appName, instances: 1, compose: [appComponent(appName)] }); + await installOnNodes(env, app, [holder]); + const { hash, ...withoutHash } = await dbClient(holder + 1).localApp(appName); + expect(hash, 'fixture: the app is installed with a hash').to.be.a('string'); + const unhashed = `${appName}unhashed`; + const afterIds = env.clients.map((c) => c.getLastEventId()); + await dbClient(holder + 1).seedLocalApp({ ...withoutHash, name: unhashed }); + return { unhashed, afterIds }; + } + const storedFrom = (i, afterId, holder, appName) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(socketAddr(holder + 1)) && d.apps?.some((a) => a.name === appName), + 120000, + { afterId }, + ); + + before(async function () { + this.timeout(420000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the FluxOS the nodes upgrade from: build it from that tree with FLUX_E2E_TAG= ./test-infra/build-images.sh fluxos-01'); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + tickerAutostart: false, + nodeImageTags: { [NOT_UPGRADED]: BASE_TAG }, + }); + if (BASE_TAG === env.nodeImageTag(UPGRADED[0])) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG'); + await bootAndPeer(env); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('an upgraded node leaves the app out, and the node not yet upgraded goes on storing the rest', async function () { + this.timeout(420000); + const appName = `e2erollupgraded${stamp}`; + const holder = UPGRADED[0]; + const { unhashed, afterIds } = await holderWithUnhashedApp(appName, holder); + + const { data: announced } = await env.clients[holder].waitForEvent( + 'app:running', (d) => d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[holder] }, + ); + expect(announced.unannounced, 'the upgraded holder announced the app it has no hash for').to.deep.equal([unhashed]); + await storedFrom(NOT_UPGRADED, afterIds[NOT_UPGRADED], holder, appName) + .catch(() => { throw new Error('the node not yet upgraded stored no announcement from the upgraded holder since the unhashed app was installed'); }); + }); + + it('a node not yet upgraded announces the app, and the upgraded nodes store the rest and refuse it', async function () { + this.timeout(420000); + const appName = `e2erollbase${stamp}`; + const holder = NOT_UPGRADED; + const { unhashed, afterIds } = await holderWithUnhashedApp(appName, holder); + + await Promise.all(UPGRADED.map(async (i) => { + const { data: stored } = await storedFrom(i, afterIds[i], holder, appName) + .catch(() => { throw new Error(`upgraded node ${i} stored no announcement from the node not yet upgraded since the unhashed app was installed`); }); + expect(stored.apps.map((a) => a.name)).to.include(appName); + await env.clients[i].waitForEvent( + 'network:apprunningRefused', + (d) => d.refusals?.some((r) => bare(r.ip) === bare(socketAddr(holder + 1)) && r.names.includes(unhashed)), + 10000, + { afterId: afterIds[i] }, + ); + expect(await dbClient(i + 1).appLocationAt(unhashed, socketAddr(holder + 1)), `upgraded node ${i} holds a row for the app without a hash`).to.equal(null); + })); + }); +}); From 94137bc3b1a3c5a140b2f6e6e79cb3e564213e84 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:06:56 +0100 Subject: [PATCH 339/369] fix(apps): a test install removes only what it installed A test install tears down the app it wrote, and nothing else: a call refused before it wrote the app's row - an app already installed here, a check that stops it - leaves the node's apps as it found them, and still clears its test-install mark. testappinstall refuses an app this node holds before it takes the node. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appInstaller.js | 32 +++++++++++----- tests/unit/appInstaller.test.js | 37 +++++++++++++++++-- 2 files changed, 56 insertions(+), 13 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 47df1ea822..9a4368ff4e 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -683,16 +683,18 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f // costs the node every install, redeploy, spawn and reinstall pass it is offered // until FluxOS restarts. if (acquired) globalState.installationInProgress = false; - // Bound to the hold: a call that did not take it never wrote the app's row - // and never set the mark, so it has neither to take back. The mark is set - // under the hold with nothing awaited in between, so holding it is the same - // fact as having marked. + // The mark is bound to the hold: it is set under the hold with nothing awaited + // in between, so a call that took the hold has a mark to take back. The + // teardown is bound to the row: a call that never wrote one created nothing, + // and an app a refusal finds already installed is not its to remove. if (test && acquired) { - try { - await appUninstaller.removeAppLocally(appSpecs.name, null, true, false, false); - log.info(`Test cleanup completed for ${appSpecs.name}`); - } catch (cleanupError) { - log.error(`Error during test cleanup for ${appSpecs.name}: ${cleanupError.message}`); + if (claimed) { + try { + await appUninstaller.removeAppLocally(appSpecs.name, null, true, false, false); + log.info(`Test cleanup completed for ${appSpecs.name}`); + } catch (cleanupError) { + log.error(`Error during test cleanup for ${appSpecs.name}: ${cleanupError.message}`); + } } // Released after the teardown, not before: the row is what the announcement // reads, and it exists until the teardown has deleted it. @@ -1278,6 +1280,18 @@ async function testAppInstall(req, res) { appSpecifications.hash = hash; } + // An app this node holds is not tested here: the test would install over it. + const installed = await dbHelper.findOneInDatabase( + dbHelper.databaseConnection().db(config.database.appslocal.database), + localAppsInformation, + { name: appSpecifications.name }, + { projection: { _id: 0, name: 1 } }, + ); + if (installed) { + res.json(messageHelper.createErrorMessage(`Flux App ${appSpecifications.name} is installed on this node. Test it on a node that does not hold it.`)); + return; + } + // Test installation - similar to regular install but with test flag // Skip all requirement checks for test installations (geolocation, static IP, hardware, nodes) await checkAppRequirements(appSpecifications, true, true, true); diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 80602d23c3..53cab33cee 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -112,7 +112,9 @@ describe('appInstaller tests', () => { }; dbHelperStub = { - databaseConnection: sinon.stub(), + // A connection always hands back a client; a test that needs a database + // to answer says what it answers. + databaseConnection: sinon.stub().returns({ db: sinon.stub().returns('database') }), findInDatabase: sinon.stub(), findOneInDatabase: sinon.stub(), insertOneToDatabase: sinon.stub(), @@ -642,6 +644,19 @@ describe('appInstaller tests', () => { expect(pastTheGate()).to.equal(true); }); + it('refuses to test an app this node holds, before it takes the node', async () => { + const req = requestAs(owner); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findOneInDatabase.resolves({ name: 'trialapp' }); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate(), 'the test went on to install over the app').to.equal(false); + expect(res.json.firstCall.args[0].data.message).to.equal('Flux App trialapp is installed on this node. Test it on a node that does not hold it.'); + }); + it('refuses a caller with no session at all', async () => { const req = { params: { appname: 'temphash' }, query: {} }; const res = response(); @@ -1367,14 +1382,15 @@ describe('appInstaller tests', () => { // has: the row is written before the image is fetched, so the claim exists by // the time the install gives up. `failBeforeRow` moves the failure ahead of // the row instead. - function buildFailingInstaller(removeAppLocallyStub, { failBeforeRow = false } = {}) { + function buildFailingInstaller(removeAppLocallyStub, { failBeforeRow = false, alreadyInstalled = false } = {}) { const dbHelperStubLocal = { databaseConnection: sinon.stub().returns({ db: () => ({ collection: () => ({}) }) }), findInDatabase: sinon.stub().resolves([]), - // 1st call = "already installed?" -> null so the install proceeds. + // 1st call = "already installed?" -> null so the install proceeds, unless + // the app is already installed here. findOneInDatabase: (() => { const stub = sinon.stub().resolves({ name: 'testapp' }); - stub.onFirstCall().resolves(null); + stub.onFirstCall().resolves(alreadyInstalled ? { name: 'testapp' } : null); return stub; })(), findOneAndDeleteInDatabase: sinon.stub().resolves(), @@ -1634,6 +1650,19 @@ describe('appInstaller tests', () => { }); }); + // A test install refused because the app is already installed here created + // nothing, and the installed app is not its to remove. + it('leaves an app already installed here alone when a test install of it is refused', async () => { + const removeAppLocallyStub = sinon.stub().resolves(); + const appInstallerWithDb = buildFailingInstaller(removeAppLocallyStub, { alreadyInstalled: true }); + + const outcome = await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, true); + + expect(outcome, 'never reached the refusal this test is about').to.equal(InstallOutcome.ALREADY_INSTALLED); + expect(removeAppLocallyStub.called, 'a refused test install removed the app installed here').to.equal(false); + expect(globalStateStub.testInstallingApps.has('testapp'), 'left the mark behind').to.equal(false); + }); + it('runs the post-install broadcast only AFTER releasing the install lock', async () => { // The announcement runs for as long as a broadcast cycle takes, and every // other install, removal and redeploy on this node refuses while the install From 913df1cd3c7b1bd36231d513b229e91de8211c32 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:08:54 +0100 Subject: [PATCH 340/369] fix(apps): a redeploy that ends without the app drops its placement record A redeploy keeps this node's record of when it took the app for the reinstall. A soft redeploy that removed the app and did not install it again - the reinstall refused, or the database stopping it - drops the record, so a later placement here starts a new one. The redeploy for a change of components is rebuildApp, which drops the record and tells the network whenever the app does not come back, a failed requirement check after the removal included. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 38 +++++----- tests/unit/advancedWorkflows.test.js | 69 ++++++++++++++++++- 2 files changed, 85 insertions(+), 22 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 5363f4a98a..4614b677d9 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -1538,6 +1538,18 @@ async function mayTearDownToRebuild(appSpecs, res) { * @param {object} appSpecs - App specifications * @param {object} res - Response object */ +/** + * Drop this node's record of when it took an app a redeploy removed and did not + * install again. The removal kept the record for the reinstall; without one the + * app has left this node, and a later placement here is a new one. + * @param {string} appName + * @returns {Promise} + */ +async function forgetPlacementOfAppNotReinstalled(appName) { + await appPlacements.removePlacement(appName) + .catch((error) => log.error(`the placement of ${appName} was not removed: ${error.message}`)); +} + async function softRedeploy(appSpecs, res) { // Whether softRemoveAppLocally ran to completion. False means the removal did // not FINISH - which is not the same as "it never started": softRemoveAppLocally @@ -1644,6 +1656,7 @@ async function softRedeploy(appSpecs, res) { res.write(serviceHelper.ensureString(notReinstalled)); if (res.flush) res.flush(); } + await forgetPlacementOfAppNotReinstalled(appSpecs.name); globalState.softRedeployInProgress = false; return; } @@ -1686,6 +1699,7 @@ async function softRedeploy(appSpecs, res) { res.write(serviceHelper.ensureString(notReinstalled)); if (res.flush) res.flush(); } + await forgetPlacementOfAppNotReinstalled(appSpecs.name); return; } log.warn(`REMOVAL REASON: Soft redeploy failure - ${appSpecs.name} failed during soft redeploy: ${error.message} (softRedeploy)`); @@ -4499,29 +4513,15 @@ async function reinstallOldApplications() { log.warn('Performing full hard redeploy to handle component changes...'); log.warn(`REMOVAL REASON: Component structure change (v8+) - ${appSpecifications.name} component count/names changed`); + // A hard redeploy: removed and installed again as a whole. One that does + // not bring the app back tells the network and drops this node's record + // of when it took the app. // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appSpecifications.name, null, false, false, false, { redeploying: true }); - const appRedeployResponse = messageHelper.createSuccessMessage('Application removed. Awaiting installation...'); - log.info(appRedeployResponse); - - // verify requirements - // eslint-disable-next-line no-await-in-loop - await appInstaller.checkAppRequirements(appSpecifications); - - // register - // eslint-disable-next-line no-await-in-loop - const outcome = await appInstaller.registerAppLocally(appSpecifications, undefined, null, false, true); + const outcome = await appRebuild.rebuildApp(appSpecifications); if (outcome === InstallOutcome.INSTALLED) { log.info(`Application ${appSpecifications.name} redeployed with new component structure`); } else { - // The removal above deleted the local row, and only a successful - // install writes it back. Nothing reconciles an app with no row, - // so this node has simply lost the instance until it is placed - // here again. - log.error(`Application ${appSpecifications.name} was removed for a component structure change and NOT reinstalled (${outcome}). This node no longer holds it and cannot recover it on its own.`); - // eslint-disable-next-line no-await-in-loop - await appPlacements.removePlacement(appSpecifications.name) - .catch((error) => log.error(`the placement of ${appSpecifications.name} was not removed: ${error.message}`)); + log.error(`Application ${appSpecifications.name} was not redeployed for a component structure change (${outcome}).`); } // eslint-disable-next-line no-continue diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index 43632d6087..a6f51ac8dd 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -13,6 +13,7 @@ const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileg const dbHelper = require('../../ZelBack/src/services/dbHelper'); const appsRuntimeState = require('../../ZelBack/src/services/appManagement/appsRuntimeState'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); +const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutcome'); const { MongoNetworkTimeoutError } = require('mongodb'); const log = require('../../ZelBack/src/lib/log'); const https = require('https'); @@ -6916,6 +6917,52 @@ describe('advancedWorkflows tests', () => { expect(globalState.softRedeployInProgress).to.be.false; }); + // A redeploy keeps the node's record of when it took the app for the + // reinstall; one that ends without the app drops it. + it('drops the placement record when the app was removed and the reinstall did not happen', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); + sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); + // the node is taken by another install by the time the reinstall asks for it + sinon.stub(appInstaller, 'checkAppRequirements').callsFake(async () => { + globalState.installationInProgress = true; + return true; + }); + const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); + + try { + await advancedWorkflows.softRedeploy(v3Spec, null); + } finally { + globalState.installationInProgress = false; + } + + sinon.assert.calledOnceWithExactly(removePlacement, 'TestApp'); + }); + + it('drops the placement record when the step after the removal cannot read the database', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); + sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); + + await advancedWorkflows.softRedeploy(v3Spec, null); + + sinon.assert.calledOnceWithExactly(removePlacement, 'TestApp'); + }); + + it('keeps the placement record when the removal never finished', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); + sinon.stub(appUninstaller, 'softUninstallApplication').rejects(new Error('Container fluxTestApp not found')); + sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); + + await advancedWorkflows.softRedeploy(v3Spec, null); + + sinon.assert.notCalled(removePlacement); + }); + it('records that a failure after the app is down still force-uninstalls it', async () => { // Pins CURRENT behaviour, not desired behaviour. The gate covers a removal // that never finished; past that point a failure still force-uninstalls @@ -7567,11 +7614,15 @@ describe('advancedWorkflows tests', () => { let registerAppLocally; let removePlacement; + let checkAppRequirements; + let announceRemoval; + beforeEach(() => { dbHelper.findInDatabase.resolves([v8Installed]); dbHelper.findOneInDatabase.resolves(v8New); - sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); - removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + checkAppRequirements = sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(RemovalOutcome.REMOVED); + announceRemoval = sinon.stub(appUninstaller, 'announceRemoval').resolves(); registerAppLocally = sinon.stub(appInstaller, 'registerAppLocally').resolves(InstallOutcome.INSTALLED); removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); }); @@ -7585,12 +7636,24 @@ describe('advancedWorkflows tests', () => { sinon.assert.notCalled(removePlacement); }); - it('drops the record when the reinstall does not bring the app back', async () => { + it('drops the record when the reinstall does not bring the app back, and tells the network', async () => { registerAppLocally.resolves(InstallOutcome.FAILED); await advancedWorkflows.reinstallOldApplications(); sinon.assert.calledOnceWithExactly(removePlacement, 'myapp'); + sinon.assert.calledOnceWithExactly(announceRemoval, 'myapp'); + }); + + it('drops the record when the new specification\'s requirements are not met after the removal', async () => { + checkAppRequirements.rejects(new Error('Insufficient RAM')); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(removeAppLocally); + sinon.assert.notCalled(registerAppLocally); + sinon.assert.calledOnceWithExactly(removePlacement, 'myapp'); + sinon.assert.calledOnceWithExactly(announceRemoval, 'myapp'); }); }); From 215546d68cee9ddebe4e20b20d1faf9ce0da3e06 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:15:14 +0100 Subject: [PATCH 341/369] feat(harness): network:apprunning says everything a node did with an announcement Published for every apprunning message a node handles from gossip, with the entries it wrote, the names it refused, whether it cleared the sender's rows, and whether it relayed the message. storeAppRunningMessage reports the same. network:apprunningRefused is the peer-sync summary only. Suites 3501 and 3505 read both sides from the one event; suite 46 reads a cleared snapshot from `cleared`. Co-Authored-By: Claude Opus 5.5 --- .../src/services/appMessaging/messageStore.js | 31 +++++++++--- ZelBack/src/services/fluxCommunication.js | 15 ++++-- ...ouncement-is-judged-one-entry-at-a-time.js | 48 ++++++++++--------- ...n-unhashed-app-across-a-rolling-upgrade.js | 11 ++--- .../46-reconciler-boot-broadcast-ordering.js | 2 +- tests/unit/messageStore.test.js | 16 +++++-- .../messageStoreAppRunningEntries.test.js | 27 +++++++++-- 7 files changed, 101 insertions(+), 49 deletions(-) diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index 46a444c22c..3b671a6569 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -313,7 +313,9 @@ function logRefusedEntries(caller, refusals) { * is never read as an empty snapshot, which deletes the sender's rows: only a * message that names no apps at all says that. * @param {object} message - Message to store - * @returns {Promise<{stored: boolean, rebroadcast: boolean}|Error>} What was stored, or Error when the message itself is malformed + * @returns {Promise<{stored: boolean, rebroadcast: boolean, written: Array, refused: Array<*>, cleared: boolean}|Error>} + * Whether anything was stored and the message is to be relayed, the entries written, the names + * refused, and whether the sender's rows were cleared; or Error when the message itself is malformed */ async function storeAppRunningMessage(message) { /* message object @@ -329,6 +331,9 @@ async function storeAppRunningMessage(message) { * @param apps array (for version 2) */ const appsMessages = []; + // What this message did here, for the caller to report: the entries written, + // the names refused, and whether the sender's rows were cleared. + let refusedNames = []; if (!message || typeof message !== 'object' || typeof message.type !== 'string' || typeof message.version !== 'number' || typeof message.broadcastedAt !== 'number' || typeof message.ip !== 'string') { return new Error('Invalid Flux App Running message for storing'); @@ -357,19 +362,23 @@ async function storeAppRunningMessage(message) { return new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode'); } const { stored, refused } = appRunningEntries(message.apps); + refusedNames = refused.map((app) => app?.name ?? null); if (refused.length) { logRefusedEntries('storeAppRunningMessage', [{ ip: message.ip, broadcastedAt: message.broadcastedAt, refused }]); - fluxEventBus.publish('network:apprunningRefused', { path: 'gossip', refusals: [{ ip: message.ip, names: refused.map((app) => app?.name ?? null) }] }); } if (message.apps.length && !stored.length) { - return { stored: false, rebroadcast: false }; + return { + stored: false, rebroadcast: false, written: [], refused: refusedNames, cleared: false, + }; } appsMessages.push(...stored); } if (message.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) { log.warn(`Rejecting old/not valid Fluxapprunning message, message:${JSON.stringify(message)}`); - return { stored: false, rebroadcast: false }; + return { + stored: false, rebroadcast: false, written: [], refused: refusedNames, cleared: false, + }; } const db = dbHelper.databaseConnection(); @@ -378,6 +387,7 @@ async function storeAppRunningMessage(message) { const [ip] = await registryManager.currentAddressesOf([{ ip: message.ip, at: message.broadcastedAt }]); let anyStored = false; + const written = []; const incomingDate = new Date(message.broadcastedAt); const isNewer = { $gt: [incomingDate, { $ifNull: ['$broadcastedAt', new Date(0)] }] }; @@ -407,6 +417,7 @@ async function storeAppRunningMessage(message) { ); if (result.modifiedCount > 0 || result.upsertedCount > 0) { anyStored = true; + written.push(app); } } @@ -420,7 +431,9 @@ async function storeAppRunningMessage(message) { await dbHelper.removeDocumentsFromCollection(database, appsInstallingBroadcasts, { 'data.ip': message.ip }); anyStored = true; } else { - return { stored: false, rebroadcast: false }; + return { + stored: false, rebroadcast: false, written: [], refused: refusedNames, cleared: false, + }; } } @@ -432,7 +445,13 @@ async function storeAppRunningMessage(message) { await dbHelper.removeDocumentsFromCollection(database, appsInstallingBroadcasts, { 'data.name': app.name, 'data.ip': message.ip }); } - return { stored: anyStored, rebroadcast: anyStored }; + return { + stored: anyStored, + rebroadcast: anyStored, + written, + refused: refusedNames, + cleared: message.version === 2 && appsMessages.length === 0 && anyStored, + }; } /** diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 2f678fb3f2..066b4865d0 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -470,14 +470,21 @@ async function handleAppRunningMessage(message, fromIP, port) { log.warn(`handleAppRunningMessage - not stored, from ${fromIP}:${port} for ${message.data?.ip}: ${result.message}`); return; } - if (result.stored) { - fluxEventBus.publish('network:apprunning', { ip: message.data.ip, apps: message.data.apps || [{ name: message.data.name }] }); - } const currentTimeStamp = Date.now(); const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(message, currentTimeStamp, 240000); - if (result.rebroadcast && timestampOK) { + const relayed = Boolean(result.rebroadcast && timestampOK); + if (relayed) { announceToPeers(message, `${fromIP}:${port}`); } + // Everything this node did with the message: the entries it wrote, the names + // it refused, whether it cleared the sender's rows, and whether it passed it on. + fluxEventBus.publish('network:apprunning', { + ip: message.data.ip, + apps: result.written, + refused: result.refused, + cleared: result.cleared, + relayed, + }); } catch (error) { log.error(error); } diff --git a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js index 4a3cc47e3e..fb9a7e518c 100644 --- a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js +++ b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js @@ -15,9 +15,10 @@ import { socketAddr } from '../framework/state-events.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // A version 2 apprunning message names every app its sender holds. Each entry is -// judged on its own: one without a name and a hash is refused -// (network:apprunningRefused) and the rest of the message is stored -// (network:apprunning) and relayed as if it were not there. +// judged on its own: one without a name and a hash is refused and the rest of the +// message is stored and relayed as if it were not there. Each node says what it +// did with a message in one network:apprunning event: the entries it wrote, the +// names it refused, whether it cleared the sender's rows, and whether it relayed. // // The stub peer is a listed node with its own key. Nodes 1 and 2 cannot reach it, // so they hear what it says only through node 0's relay, and a message node 0 @@ -60,19 +61,18 @@ describe('an announcement is judged one entry at a time', function () { const stubIp = () => getSubnetConfig().nodeIp(STUB_INDEX + 1); const rowNamesAt = async (i, address) => (await dbClient(i + 1).getAppLocationsByIp(address)).map((row) => row.name); const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); - // Every node stored a message from `ip` naming `name`, after `afterIds`. - const everyNodeStored = (afterIds, ip, name, timeout = 60000) => Promise.all(realNodes.map((i) => env.clients[i].waitForEvent( + // What each node did with the message from `ip` that names `name`, after `afterIds`. + const handledBy = (nodes, afterIds, ip, name, timeout = 60000) => Promise.all(nodes.map(async (i) => (await env.clients[i].waitForEvent( 'network:apprunning', - (d) => bare(d.ip) === bare(ip) && d.apps?.some((a) => a.name === name), + (d) => bare(d.ip) === bare(ip) && [...(d.apps ?? []).map((a) => a.name), ...(d.refused ?? [])].includes(name), timeout, { afterId: afterIds[i] }, - ))); - const refusedOn = (i, afterId, ip, name, timeout = 60000) => env.clients[i].waitForEvent( - 'network:apprunningRefused', - (d) => d.path === 'gossip' && d.refusals?.some((r) => bare(r.ip) === bare(ip) && r.names.includes(name)), - timeout, - { afterId }, - ); + )).data)); + const everyNodeStored = async (afterIds, ip, name) => { + const handled = await handledBy(realNodes, afterIds, ip, name); + handled.forEach((d, i) => expect(d.apps.map((a) => a.name), `node ${i} did not store ${name}`).to.include(name)); + return handled; + }; const message = (apps) => ({ type: 'fluxapprunning', version: 2, apps, ip: stubIp(), broadcastedAt: Date.now(), osUptime: 10000, staticIp: false, }); @@ -121,11 +121,12 @@ describe('an announcement is judged one entry at a time', function () { ])); expect(sent.sent, 'fixture: the message reached node 0 and no other').to.equal(1); - const stored = await everyNodeStored(afterIds, stubIp(), kept); - stored.forEach(({ data }, i) => { - expect(data.apps.map((a) => a.name), `node ${i} stored`).to.include(kept); + const handled = await everyNodeStored(afterIds, stubIp(), kept); + handled.forEach((d, i) => { + expect(d.refused, `node ${i} did not refuse the entry without a hash`).to.deep.equal([refused]); + expect(d.apps.map((a) => a.name), `node ${i} wrote the entry without a hash`).to.not.include(refused); }); - await Promise.all(realNodes.map((i) => refusedOn(i, afterIds[i], stubIp(), refused))); + expect(handled[0].relayed, 'node 0 did not relay the message to the nodes behind it').to.equal(true); // eslint-disable-next-line no-restricted-syntax for (const i of realNodes) { // eslint-disable-next-line no-await-in-loop @@ -152,10 +153,11 @@ describe('an announcement is judged one entry at a time', function () { await stub.runApp(marker, { hash: `hash-${marker}` }); await everyNodeStored(afterIds, stubIp(), marker); - await refusedOn(0, afterIds[0], stubIp(), refusedOnly); + const [onNodeZero] = await handledBy([0], afterIds, stubIp(), refusedOnly); + expect(onNodeZero).to.deep.include({ apps: [], refused: [refusedOnly], cleared: false, relayed: false }); // Relayed, it would have reached them before the marker did. - await Promise.all(BEHIND_RELAY.map((i) => assertNoEvent(env.clients[i], 'network:apprunningRefused', - (d) => d.refusals?.some((r) => r.names.includes(refusedOnly)), 0, { afterId: afterIds[i] }))); + await Promise.all(BEHIND_RELAY.map((i) => assertNoEvent(env.clients[i], 'network:apprunning', + (d) => d.refused?.includes(refusedOnly), 0, { afterId: afterIds[i] }))); // eslint-disable-next-line no-restricted-syntax for (const i of realNodes) { // eslint-disable-next-line no-await-in-loop @@ -175,7 +177,7 @@ describe('an announcement is judged one entry at a time', function () { afterIds = marks(); await stub.broadcast(message([])); await Promise.all(realNodes.map((i) => env.clients[i].waitForEvent( - 'network:apprunning', (d) => bare(d.ip) === bare(stubIp()) && d.apps?.length === 0, 60000, { afterId: afterIds[i] }, + 'network:apprunning', (d) => bare(d.ip) === bare(stubIp()) && d.cleared === true, 60000, { afterId: afterIds[i] }, ))); // eslint-disable-next-line no-restricted-syntax for (const i of realNodes) { @@ -216,7 +218,7 @@ describe('an announcement is judged one entry at a time', function () { await Promise.all(others.map((i) => env.clients[i].waitForEvent( 'network:apprunning', (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[i] }, ).catch(() => { throw new Error(`node ${i} stored no announcement from the holder since the unhashed app was installed`); }))); - await Promise.all(others.map((i) => assertNoEvent(env.clients[i], 'network:apprunningRefused', - (d) => d.refusals?.some((r) => bare(r.ip) === bare(holder)), 0, { afterId: afterIds[i] }))); + await Promise.all(others.map((i) => assertNoEvent(env.clients[i], 'network:apprunning', + (d) => bare(d.ip) === bare(holder) && d.refused?.length > 0, 0, { afterId: afterIds[i] }))); }); }); diff --git a/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js b/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js index b1af283bd5..d3a391ae29 100644 --- a/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js +++ b/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js @@ -20,8 +20,8 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // An upgraded node holding an app installed without a hash leaves it out of its // announcements (app:running.unannounced), so the node not yet upgraded goes on // storing the rest (network:apprunning). A node not yet upgraded announces such -// an app, and the upgraded nodes store its other entries and refuse that one -// (network:apprunningRefused). +// an app, and the upgraded nodes write its other entries and refuse that one, +// both of which their network:apprunning says. // // Without FLUX_E2E_BASE_TAG the suite fails before booting, naming what it needs. @@ -116,12 +116,7 @@ describe('an app installed without a hash, across a rolling upgrade', function ( const { data: stored } = await storedFrom(i, afterIds[i], holder, appName) .catch(() => { throw new Error(`upgraded node ${i} stored no announcement from the node not yet upgraded since the unhashed app was installed`); }); expect(stored.apps.map((a) => a.name)).to.include(appName); - await env.clients[i].waitForEvent( - 'network:apprunningRefused', - (d) => d.refusals?.some((r) => bare(r.ip) === bare(socketAddr(holder + 1)) && r.names.includes(unhashed)), - 10000, - { afterId: afterIds[i] }, - ); + expect(stored.refused, `upgraded node ${i} did not refuse the app without a hash`).to.deep.equal([unhashed]); expect(await dbClient(i + 1).appLocationAt(unhashed, socketAddr(holder + 1)), `upgraded node ${i} holds a row for the app without a hash`).to.equal(null); })); }); diff --git a/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js b/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js index 27e4230505..280ebc8dfd 100644 --- a/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js +++ b/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js @@ -83,7 +83,7 @@ describe('first post-boot broadcast: complete, never empty, never destructive', // buffer holds every apprunning it received across the reboot window) const received = peerClient.getEventBuffer().filter((e) => e.id > afterId && e.event === 'network:apprunning'); expect(received.length, 'buffer must hold the apprunning waitForEvent just matched (filter shape drift)').to.be.greaterThan(0); - const empty = received.filter((e) => Array.isArray(e.data?.apps) && e.data.apps.length === 0); + const empty = received.filter((e) => e.data?.cleared === true); expect(empty, 'no empty fluxapprunning v2 snapshots').to.have.lengthOf(0); // (a): the peer's location rows for the app survived the whole sequence diff --git a/tests/unit/messageStore.test.js b/tests/unit/messageStore.test.js index 3cee0f9ef8..0a26089fe7 100644 --- a/tests/unit/messageStore.test.js +++ b/tests/unit/messageStore.test.js @@ -417,7 +417,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: false, rebroadcast: false }); + expect(result).to.deep.equal({ + stored: false, rebroadcast: false, written: [], refused: [], cleared: false, + }); expect(logStub.warn.called).to.be.true; }); @@ -438,7 +440,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: [{ name: 'testapp', hash: 'hash123' }], refused: [], cleared: false, + }); expect(dbHelperStub.updateOneInDatabase.calledOnce).to.be.true; }); @@ -461,7 +465,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: message.apps, refused: [], cleared: false, + }); expect(dbHelperStub.updateOneInDatabase.callCount).to.equal(2); // Should clean up installing records for each app (location + broadcast per app) expect(dbHelperStub.removeDocumentsFromCollection.callCount).to.equal(4); @@ -483,7 +489,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: [], refused: [], cleared: true, + }); // Called three times: locations, installing locations, installing broadcasts expect(dbHelperStub.removeDocumentsFromCollection.callCount).to.equal(3); }); diff --git a/tests/unit/messageStoreAppRunningEntries.test.js b/tests/unit/messageStoreAppRunningEntries.test.js index e12f0c6647..f87ab4b6e1 100644 --- a/tests/unit/messageStoreAppRunningEntries.test.js +++ b/tests/unit/messageStoreAppRunningEntries.test.js @@ -79,7 +79,13 @@ describe('apprunning entries are stored one at a time', () => { { name: 'db', hash: 'dbhash' }, ])); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, + rebroadcast: true, + written: [{ name: 'web', hash: 'webhash' }, { name: 'db', hash: 'dbhash' }], + refused: ['probe'], + cleared: false, + }); expect((await rowsAt()).map((r) => r.name)).to.deep.equal(['db', 'web']); }); @@ -105,7 +111,9 @@ describe('apprunning entries are stored one at a time', () => { it('stores nothing and relays nothing when every entry is refused', async () => { const result = await messageStore.storeAppRunningMessage(message([{ name: 'probe' }, null])); - expect(result).to.deep.equal({ stored: false, rebroadcast: false }); + expect(result).to.deep.equal({ + stored: false, rebroadcast: false, written: [], refused: ['probe', null], cleared: false, + }); expect(await rowsAt()).to.deep.equal([]); }); @@ -128,10 +136,23 @@ describe('apprunning entries are stored one at a time', () => { const result = await messageStore.storeAppRunningMessage(message([])); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: [], refused: [], cleared: true, + }); expect(await rowsAt()).to.deep.equal([]); }); + it('reports no entry as written when a message changes no row', async () => { + const at = Date.now(); + await messageStore.storeAppRunningMessage(message([{ name: 'web', hash: 'webhash' }], at)); + + const again = await messageStore.storeAppRunningMessage(message([{ name: 'web', hash: 'webhash' }], at)); + + expect(again).to.deep.equal({ + stored: false, rebroadcast: false, written: [], refused: [], cleared: false, + }); + }); + it('writes nothing for another node', async () => { await locations.insertOne(row('web', otherIp, Date.now() - 10 * 60 * 1000)); From 5935165d744e4fa9467655f81badbf9f6eab6dd0 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:32:20 +0100 Subject: [PATCH 342/369] fix(apps): the placement record follows the app's volume A hard install, which creates the app's volume, writes the record as now over any it finds; a removal, which deletes the volume, deletes it; a soft redeploy keeps the volume and leaves the record. A hard redeploy, which starts from an empty volume, therefore starts a new record. A record outliving its app is never read - only an installed app's is, and a hard install replaces it - so nothing depends on a removal's cleanup finishing, and removeAppLocally takes no redeploying option. At boot the records of apps not installed are removed first, on every path boot takes; an installed app without a record is then given one from this node's own location row before its first announcement. A soft redeploy publishes app:softRedeployed with its outcome. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/advancedWorkflows.js | 20 +--- .../src/services/appLifecycle/appInstaller.js | 4 +- .../src/services/appLifecycle/appRebuild.js | 9 +- .../appLifecycle/appStartupManager.js | 29 ++++-- .../services/appLifecycle/appUninstaller.js | 9 +- .../services/appManagement/appPlacements.js | 89 ++++++++++------- tests/unit/advancedWorkflows.test.js | 54 +++------- tests/unit/appPlacements.test.js | 98 ++++++++++++------- tests/unit/appRebuild.test.js | 47 --------- tests/unit/appStartupManager.test.js | 47 +++++++-- tests/unit/appUninstaller.test.js | 15 +-- 11 files changed, 198 insertions(+), 223 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index 4614b677d9..a761b1e7d7 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -21,7 +21,6 @@ const fluxEventBus = require('../utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../utils/addressMoveEvents'); const { InstallOutcome } = require('../utils/installOutcome'); const appRebuild = require('./appRebuild'); -const appPlacements = require('../appManagement/appPlacements'); const generalService = require('../generalService'); const placementFeasibility = require('../appPlacement/placementFeasibility'); // eslint-disable-next-line no-unused-vars @@ -1538,18 +1537,6 @@ async function mayTearDownToRebuild(appSpecs, res) { * @param {object} appSpecs - App specifications * @param {object} res - Response object */ -/** - * Drop this node's record of when it took an app a redeploy removed and did not - * install again. The removal kept the record for the reinstall; without one the - * app has left this node, and a later placement here is a new one. - * @param {string} appName - * @returns {Promise} - */ -async function forgetPlacementOfAppNotReinstalled(appName) { - await appPlacements.removePlacement(appName) - .catch((error) => log.error(`the placement of ${appName} was not removed: ${error.message}`)); -} - async function softRedeploy(appSpecs, res) { // Whether softRemoveAppLocally ran to completion. False means the removal did // not FINISH - which is not the same as "it never started": softRemoveAppLocally @@ -1656,12 +1643,13 @@ async function softRedeploy(appSpecs, res) { res.write(serviceHelper.ensureString(notReinstalled)); if (res.flush) res.flush(); } - await forgetPlacementOfAppNotReinstalled(appSpecs.name); globalState.softRedeployInProgress = false; + fluxEventBus.publish('app:softRedeployed', { name: appSpecs.name, outcome }); return; } log.info('Application softly redeployed'); globalState.softRedeployInProgress = false; + fluxEventBus.publish('app:softRedeployed', { name: appSpecs.name, outcome }); } catch (error) { log.info('Error on softRedeploy'); log.error(error); @@ -1699,7 +1687,6 @@ async function softRedeploy(appSpecs, res) { res.write(serviceHelper.ensureString(notReinstalled)); if (res.flush) res.flush(); } - await forgetPlacementOfAppNotReinstalled(appSpecs.name); return; } log.warn(`REMOVAL REASON: Soft redeploy failure - ${appSpecs.name} failed during soft redeploy: ${error.message} (softRedeploy)`); @@ -4514,8 +4501,7 @@ async function reinstallOldApplications() { log.warn(`REMOVAL REASON: Component structure change (v8+) - ${appSpecifications.name} component count/names changed`); // A hard redeploy: removed and installed again as a whole. One that does - // not bring the app back tells the network and drops this node's record - // of when it took the app. + // not bring the app back tells the network. // eslint-disable-next-line no-await-in-loop const outcome = await appRebuild.rebuildApp(appSpecifications); if (outcome === InstallOutcome.INSTALLED) { diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 9a4368ff4e..73b3f490a1 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -548,8 +548,8 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f // until its monitor has measured what it holds. folderStates.installed(appSpecifications); claimed = true; - // A redeploy reinstalls an app whose record this node kept; only a new - // placement writes one. + // The app's volumes are created next: this node's record of when it took + // the app starts now. if (!test) await appPlacements.recordPlacement(appSpecifications.name); const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; diff --git a/ZelBack/src/services/appLifecycle/appRebuild.js b/ZelBack/src/services/appLifecycle/appRebuild.js index 22b971c8b3..bdaae867e4 100644 --- a/ZelBack/src/services/appLifecycle/appRebuild.js +++ b/ZelBack/src/services/appLifecycle/appRebuild.js @@ -7,7 +7,6 @@ const log = require('../../lib/log'); const fluxEventBus = require('../utils/fluxEventBus'); const appUninstaller = require('./appUninstaller'); const appInstaller = require('./appInstaller'); -const appPlacements = require('../appManagement/appPlacements'); const { RemovalOutcome } = require('../utils/removalOutcome'); const { InstallOutcome } = require('../utils/installOutcome'); @@ -25,7 +24,7 @@ const { InstallOutcome } = require('../utils/installOutcome'); * node, FAILED when the removal did not finish. */ async function rebuildApp(appSpecs, res) { - const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false, false, { redeploying: true }); + const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); if (removal !== RemovalOutcome.REMOVED && removal !== RemovalOutcome.NOT_INSTALLED) { log.warn(`appRebuild - ${appSpecs.name} was not removed (${removal}), so it is not reinstalled`); const outcome = removal === RemovalOutcome.BUSY ? InstallOutcome.BUSY : InstallOutcome.FAILED; @@ -43,12 +42,6 @@ async function rebuildApp(appSpecs, res) { outcome = InstallOutcome.FAILED; } - // The removal kept this node's record of when it took the app, for the - // reinstall; an app that is not back here has left, and the record with it. - if (outcome !== InstallOutcome.INSTALLED) { - await appPlacements.removePlacement(appSpecs.name) - .catch((error) => log.error(`appRebuild - the placement of ${appSpecs.name} was not removed: ${error.message}`)); - } if (outcome !== InstallOutcome.INSTALLED && removal === RemovalOutcome.REMOVED) { log.warn(`appRebuild - ${appSpecs.name} was removed and not reinstalled (${outcome}); telling the network`); await appUninstaller.announceRemoval(appSpecs.name) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 7ecdb55b27..6e273a9b0c 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -397,24 +397,39 @@ async function removeAllApps(reason) { } /** - * Make this node's placement records match the apps installed here, before its + * Remove the placement records of apps not installed here, on every boot + * whichever way it goes on. A failure is logged and boot goes on: a stray record + * is never read, and the next boot tries again. + * @returns {Promise} + */ +async function tidyPlacementsOnBoot() { + try { + const installed = await getInstalledAppsFromDb(); + await appPlacements.removeStrayPlacements(installed.map((app) => app.name)); + } catch (error) { + log.error(`appStartupManager - stray placement records not removed: ${error.message}`); + } +} + +/** + * Give each installed app without a placement record one, before this node's * first announcement reads them. A failure is logged and boot goes on: an app - * without a record is announced from its location row, as it is without one at - * any time. + * without a record is announced from its location row. * @returns {Promise} */ -async function reconcilePlacementsOnBoot() { +async function backfillPlacementsOnBoot() { try { const installed = await getInstalledAppsFromDb(); const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - await appPlacements.reconcilePlacements(installed.map((app) => app.name), localSocketAddr); + await appPlacements.backfillPlacements(installed.map((app) => app.name), localSocketAddr); } catch (error) { - log.error(`appStartupManager - placement records not reconciled: ${error.message}`); + log.error(`appStartupManager - placement records not backfilled: ${error.message}`); } } async function manageAppsOnBoot(bootContext) { try { + await tidyPlacementsOnBoot(); // ahead of every container start, so no slice in use can be empty here await appSlice.stopEmptySlices(); if (bootContext.firstBoot) { @@ -475,7 +490,7 @@ async function manageAppsOnBoot(bootContext) { } log.info('appStartupManager - Daemon, DB, and node confirmed, reconciling apps'); - await reconcilePlacementsOnBoot(); + await backfillPlacementsOnBoot(); await openInstalledAppPorts(); await reconcileAppsOnBoot(); } finally { diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index ee59c37900..5efb6f8633 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -913,13 +913,9 @@ async function softUninstallApplication(appName, appId, appSpecifications, res, * @param {boolean} force - Force removal * @param {boolean} endResponse - Whether to end response * @param {boolean} sendMessage - Whether to send message to network - * @param {object} [options] - * @param {boolean} [options.redeploying] The app is removed to be installed here again, - * so this node keeps its record of when it took the app. Without it the app is - * leaving this node and the record goes with it. * @returns {Promise} One of RemovalOutcome. BUSY carries no claim about the app. */ -async function removeAppLocally(app, res, force = false, endResponse = true, sendMessage = false, { redeploying = false } = {}) { +async function removeAppLocally(app, res, force = false, endResponse = true, sendMessage = false) { // Names what this call marked as departing, and nothing else: the guards below // return through the same finally without having marked anything, and a refused // duplicate must not unmark the removal that is actually running. @@ -1153,7 +1149,8 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - if (!redeploying) await appPlacements.removePlacement(appName); + // the app's volume is gone, and the record of when this node took it with it + await appPlacements.removePlacement(appName); // The app is gone for good: nothing reconciles an app with no row, so its // removal records have no reader left. Only this full-uninstall path clears // them - softRemoveAppLocally deletes the row too, but as one step of a diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js index 73740afed7..6d7eeceb9c 100644 --- a/ZelBack/src/services/appManagement/appPlacements.js +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -5,11 +5,16 @@ const fluxEventBus = require('../utils/fluxEventBus'); // When this node took each app it holds: one record per app, keyed by name. // -// Written when an app is installed here and kept through every redeploy and spec -// update, which remove and reinstall the app without it leaving the node. Deleted -// when the app leaves. The announcer reads runningSince from here and nowhere -// else, because the network's copy of this node's own location rows lapses in -// normal running - expiry, eviction, sigterm - while the app goes on running. +// The record follows the app's data on this node. It is written when a hard +// install creates the app's volume, and deleted when a removal deletes the +// volume; a soft redeploy keeps the volume and leaves the record as it is. So a +// hard redeploy, which starts from an empty volume, starts a new record. +// +// The announcer reads runningSince from here and nowhere else, because the +// network's copy of this node's own location rows lapses in normal running - +// expiry, eviction, sigterm - while the app goes on running. A record outliving +// its app is never read: only an installed app's record is, and a hard install +// overwrites whatever it finds. // // Kept apart from the app's specification in appsInformation: a spec update // deletes and re-inserts that, and this record must outlive it. @@ -33,25 +38,21 @@ async function prepareCollection() { } /** - * Record that this node took an app at `placedAt`, unless a record for it exists. + * Record that this node took an app now, over any record it had of the app: a + * hard install has just created the app's volume. * @param {string} name App name. - * @param {Date} [placedAt] When this node took it. - * @param {string} [source] What the time was taken from: install, locationRow or now. * @returns {Promise} */ -async function recordPlacement(name, placedAt = new Date(), source = 'install') { - // Two writers racing the first insert leave one record: the server retries an - // upsert that collides on a unique index matched by equality. - const result = await dbHelper.updateOneInDatabase( +async function recordPlacement(name) { + const placedAt = new Date(); + await dbHelper.updateOneInDatabase( localDatabase(), appsPlacements, { name }, - { $setOnInsert: { name, placedAt } }, + { $set: { name, placedAt } }, { upsert: true }, ); - if (result?.upsertedCount) { - fluxEventBus.publish('app:placementRecorded', { name, placedAt: placedAt.toISOString(), source }); - } + fluxEventBus.publish('app:placementRecorded', { name, placedAt: placedAt.toISOString(), source: 'install' }); } /** @@ -70,7 +71,7 @@ async function placementsOf(names) { } /** - * Forget an app this node no longer holds. + * Forget an app whose volume this node has deleted. * @param {string} name App name. * @returns {Promise} */ @@ -80,19 +81,28 @@ async function removePlacement(name) { } /** - * Make the records match the apps installed here, once, before this node first - * announces them. - * - * An installed app with no record gets one dated at the runningSince this node's - * own location row holds, so it keeps the place among its app's instances the - * network already gives it, and dated now when there is no row. A record for an - * app not installed here is removed, so a later install starts a new one. + * Remove the records of apps not installed here. + * @param {Array} installedNames Names of the apps installed here. + * @returns {Promise} How many were removed. + */ +async function removeStrayPlacements(installedNames) { + const removal = await dbHelper.removeDocumentsFromCollection(localDatabase(), appsPlacements, { name: { $nin: installedNames } }); + const removed = removal?.deletedCount ?? 0; + if (removed) log.info(`appPlacements - removed ${removed} record(s) of apps not installed here`); + fluxEventBus.publish('app:placementsTidied', { removed }); + return removed; +} + +/** + * Give each installed app without a record one, before this node first announces + * it. It is dated at the runningSince this node's own location row holds, so the + * app keeps the place among its instances the network already gives it, and + * dated now when there is no row. A record that exists is left as it is. * @param {Array} installedNames Names of the apps installed here. * @param {string} localSocketAddr This node's address. - * @returns {Promise<{created: number, removed: number}>} What changed. + * @returns {Promise<{created: number, fromLocationRows: number}>} What was written. */ -async function reconcilePlacements(installedNames, localSocketAddr) { - const database = localDatabase(); +async function backfillPlacements(installedNames, localSocketAddr) { const recorded = await placementsOf(installedNames); const missing = installedNames.filter((name) => !recorded.has(name)); @@ -108,17 +118,25 @@ async function reconcilePlacements(installedNames, localSocketAddr) { const now = new Date(); // eslint-disable-next-line no-restricted-syntax for (const name of missing) { + const placedAt = rowSince.get(name) ?? now; // eslint-disable-next-line no-await-in-loop - await recordPlacement(name, rowSince.get(name) ?? now, rowSince.has(name) ? 'locationRow' : 'now'); + const result = await dbHelper.updateOneInDatabase( + localDatabase(), + appsPlacements, + { name }, + { $setOnInsert: { name, placedAt } }, + { upsert: true }, + ); + if (result?.upsertedCount) { + fluxEventBus.publish('app:placementRecorded', { name, placedAt: placedAt.toISOString(), source: rowSince.has(name) ? 'locationRow' : 'now' }); + } } - const removal = await dbHelper.removeDocumentsFromCollection(database, appsPlacements, { name: { $nin: installedNames } }); - const removed = removal?.deletedCount ?? 0; - if (missing.length || removed) { - log.info(`appPlacements - recorded ${missing.length} placement(s) (${rowSince.size} from this node's location rows), removed ${removed} for apps not installed here`); + if (missing.length) { + log.info(`appPlacements - recorded ${missing.length} placement(s), ${rowSince.size} from this node's location rows`); } - fluxEventBus.publish('app:placementsReconciled', { created: missing.length, fromLocationRows: rowSince.size, removed }); - return { created: missing.length, removed }; + fluxEventBus.publish('app:placementsBackfilled', { created: missing.length, fromLocationRows: rowSince.size }); + return { created: missing.length, fromLocationRows: rowSince.size }; } module.exports = { @@ -126,5 +144,6 @@ module.exports = { recordPlacement, placementsOf, removePlacement, - reconcilePlacements, + removeStrayPlacements, + backfillPlacements, }; diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index a6f51ac8dd..349a15dbda 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -6917,9 +6917,9 @@ describe('advancedWorkflows tests', () => { expect(globalState.softRedeployInProgress).to.be.false; }); - // A redeploy keeps the node's record of when it took the app for the - // reinstall; one that ends without the app drops it. - it('drops the placement record when the app was removed and the reinstall did not happen', async () => { + // A soft redeploy keeps the app's volume, and with it the node's record of + // when it took the app, whichever way it ends. + it('leaves the placement record alone when the app was removed and the reinstall did not happen', async () => { sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); @@ -6929,6 +6929,7 @@ describe('advancedWorkflows tests', () => { return true; }); const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); + const recordPlacement = sinon.stub(appPlacements, 'recordPlacement').resolves(); try { await advancedWorkflows.softRedeploy(v3Spec, null); @@ -6936,31 +6937,8 @@ describe('advancedWorkflows tests', () => { globalState.installationInProgress = false; } - sinon.assert.calledOnceWithExactly(removePlacement, 'TestApp'); - }); - - it('drops the placement record when the step after the removal cannot read the database', async () => { - sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); - sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); - sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); - sinon.stub(appUninstaller, 'removeAppLocally').resolves(); - const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); - - await advancedWorkflows.softRedeploy(v3Spec, null); - - sinon.assert.calledOnceWithExactly(removePlacement, 'TestApp'); - }); - - it('keeps the placement record when the removal never finished', async () => { - sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); - sinon.stub(appUninstaller, 'softUninstallApplication').rejects(new Error('Container fluxTestApp not found')); - sinon.stub(appUninstaller, 'removeAppLocally').resolves(); - const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); - - await advancedWorkflows.softRedeploy(v3Spec, null); - sinon.assert.notCalled(removePlacement); + sinon.assert.notCalled(recordPlacement); }); it('records that a failure after the app is down still force-uninstalls it', async () => { @@ -7600,9 +7578,8 @@ describe('advancedWorkflows tests', () => { ).to.be.false; }); - // A version 8 app whose components change is removed and installed again on - // this node, so the record of when the node took it is kept - unless the - // reinstall does not bring the app back. + // A version 8 app whose components change is removed and installed again as + // a whole, and the network is told when it does not come back. describe('a redeploy for a change of components', () => { const v8Installed = { ...installedApp, version: 8, enterprise: '' }; const v8New = { @@ -7612,8 +7589,6 @@ describe('advancedWorkflows tests', () => { }; let removeAppLocally; let registerAppLocally; - let removePlacement; - let checkAppRequirements; let announceRemoval; @@ -7624,35 +7599,30 @@ describe('advancedWorkflows tests', () => { removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(RemovalOutcome.REMOVED); announceRemoval = sinon.stub(appUninstaller, 'announceRemoval').resolves(); registerAppLocally = sinon.stub(appInstaller, 'registerAppLocally').resolves(InstallOutcome.INSTALLED); - removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); }); - it('removes the app as one being redeployed, and keeps its record', async () => { + it('removes the app and installs it again', async () => { await advancedWorkflows.reinstallOldApplications(); - expect(removeAppLocally.calledOnce, 'the fixture never reached the redeploy it is about').to.equal(true); - expect(removeAppLocally.firstCall.args[5]).to.deep.equal({ redeploying: true }); - sinon.assert.calledOnce(registerAppLocally); - sinon.assert.notCalled(removePlacement); + sinon.assert.callOrder(removeAppLocally, checkAppRequirements, registerAppLocally); + sinon.assert.notCalled(announceRemoval); }); - it('drops the record when the reinstall does not bring the app back, and tells the network', async () => { + it('tells the network when the reinstall does not bring the app back', async () => { registerAppLocally.resolves(InstallOutcome.FAILED); await advancedWorkflows.reinstallOldApplications(); - sinon.assert.calledOnceWithExactly(removePlacement, 'myapp'); sinon.assert.calledOnceWithExactly(announceRemoval, 'myapp'); }); - it('drops the record when the new specification\'s requirements are not met after the removal', async () => { + it('tells the network when the new specification\'s requirements are not met after the removal', async () => { checkAppRequirements.rejects(new Error('Insufficient RAM')); await advancedWorkflows.reinstallOldApplications(); sinon.assert.calledOnce(removeAppLocally); sinon.assert.notCalled(registerAppLocally); - sinon.assert.calledOnceWithExactly(removePlacement, 'myapp'); sinon.assert.calledOnceWithExactly(announceRemoval, 'myapp'); }); }); diff --git a/tests/unit/appPlacements.test.js b/tests/unit/appPlacements.test.js index ba86df70df..eb07f1c903 100644 --- a/tests/unit/appPlacements.test.js +++ b/tests/unit/appPlacements.test.js @@ -36,26 +36,38 @@ describe('appPlacements', () => { expect(indexes.find((index) => index.name === 'name_unique')).to.include({ unique: true }); }); + // A record written as an install writes it: dated at the call, over whatever + // record the app had. + const seed = (name, placedAt) => placements.insertOne({ name, placedAt }); + describe('recordPlacement', () => { - it('records when this node took an app', async () => { - await appPlacements.recordPlacement('web', june); + it('records that this node took an app now', async () => { + const before = Date.now(); + await appPlacements.recordPlacement('web'); - expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + const [record] = await recorded(); + expect(record.name).to.equal('web'); + expect(record.placedAt.getTime()).to.be.within(before, Date.now()); }); - // A redeploy reinstalls an app whose record this node kept. - it('never moves a record that exists', async () => { - await appPlacements.recordPlacement('web', june); - await appPlacements.recordPlacement('web', july); + // A hard install creates the app's volume: the record of the data before it + // describes data this node no longer holds. + it('replaces a record the app already had', async () => { + await seed('web', june); + const before = Date.now(); - expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + await appPlacements.recordPlacement('web'); + + const records = await recorded(); + expect(records).to.have.length(1); + expect(records[0].placedAt.getTime()).to.be.within(before, Date.now()); }); it('keeps one record when two writers race the first one', async () => { await Promise.all([ - appPlacements.recordPlacement('web', june), - appPlacements.recordPlacement('web', july), - appPlacements.recordPlacement('web', june), + appPlacements.recordPlacement('web'), + appPlacements.recordPlacement('web'), + appPlacements.recordPlacement('web'), ]); expect(await recorded()).to.have.length(1); @@ -64,8 +76,8 @@ describe('appPlacements', () => { describe('placementsOf', () => { it('answers the apps it has records for, and only those', async () => { - await appPlacements.recordPlacement('web', june); - await appPlacements.recordPlacement('db', july); + await seed('web', june); + await seed('db', july); const found = await appPlacements.placementsOf(['web', 'cache']); @@ -75,8 +87,8 @@ describe('appPlacements', () => { describe('removePlacement', () => { it('forgets the one app', async () => { - await appPlacements.recordPlacement('web', june); - await appPlacements.recordPlacement('db', july); + await seed('web', june); + await seed('db', july); await appPlacements.removePlacement('web'); @@ -84,7 +96,27 @@ describe('appPlacements', () => { }); }); - describe('reconcilePlacements', () => { + describe('removeStrayPlacements', () => { + it('removes the record of an app not installed here, and only that', async () => { + await seed('gone', june); + await seed('web', july); + + const removed = await appPlacements.removeStrayPlacements(['web']); + + expect(removed).to.equal(1); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: july }]); + }); + + it('removes every record when nothing is installed', async () => { + await seed('gone', june); + + await appPlacements.removeStrayPlacements([]); + + expect(await recorded()).to.deep.equal([]); + }); + }); + + describe('backfillPlacements', () => { const row = (name, ip, runningSince) => ({ name, ip, hash: `${name}hash`, broadcastedAt: new Date(), expireAt: new Date(Date.now() + 3600000), runningSince, }); @@ -94,9 +126,9 @@ describe('appPlacements', () => { it('dates an installed app with no record at the runningSince of this node\'s own row', async () => { await locations.insertOne(row('web', me, june)); - const result = await appPlacements.reconcilePlacements(['web'], me); + const result = await appPlacements.backfillPlacements(['web'], me); - expect(result).to.deep.equal({ created: 1, removed: 0 }); + expect(result).to.deep.equal({ created: 1, fromLocationRows: 1 }); expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); }); @@ -104,7 +136,7 @@ describe('appPlacements', () => { await locations.insertOne(row('web', other, june)); const before = Date.now(); - await appPlacements.reconcilePlacements(['web'], me); + await appPlacements.backfillPlacements(['web'], me); const [{ placedAt }] = await recorded(); expect(placedAt.getTime()).to.be.within(before, Date.now()); @@ -113,7 +145,7 @@ describe('appPlacements', () => { it('dates an installed app with neither a record nor a row now', async () => { const before = Date.now(); - await appPlacements.reconcilePlacements(['web'], me); + await appPlacements.backfillPlacements(['web'], me); const [{ placedAt }] = await recorded(); expect(placedAt.getTime()).to.be.within(before, Date.now()); @@ -123,38 +155,28 @@ describe('appPlacements', () => { await locations.insertOne(row('web', me, june)); const before = Date.now(); - await appPlacements.reconcilePlacements(['web'], null); + await appPlacements.backfillPlacements(['web'], null); const [{ placedAt }] = await recorded(); expect(placedAt.getTime()).to.be.within(before, Date.now()); }); it('leaves a record it already has, whatever the row says', async () => { - await appPlacements.recordPlacement('web', june); + await seed('web', june); await locations.insertOne(row('web', me, july)); - const result = await appPlacements.reconcilePlacements(['web'], me); + const result = await appPlacements.backfillPlacements(['web'], me); - expect(result).to.deep.equal({ created: 0, removed: 0 }); + expect(result).to.deep.equal({ created: 0, fromLocationRows: 0 }); expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); }); - it('removes the record of an app not installed here', async () => { - await appPlacements.recordPlacement('gone', june); - await appPlacements.recordPlacement('web', july); - - const result = await appPlacements.reconcilePlacements(['web'], me); - - expect(result).to.deep.equal({ created: 0, removed: 1 }); - expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: july }]); - }); - - it('removes every record when nothing is installed', async () => { - await appPlacements.recordPlacement('gone', june); + it('touches no record of an app not installed here', async () => { + await seed('gone', june); - await appPlacements.reconcilePlacements([], me); + await appPlacements.backfillPlacements(['web'], me); - expect(await recorded()).to.deep.equal([]); + expect((await recorded()).map((r) => r.name)).to.deep.equal(['gone', 'web']); }); }); }); diff --git a/tests/unit/appRebuild.test.js b/tests/unit/appRebuild.test.js index c5f22917ab..cb46a8b177 100644 --- a/tests/unit/appRebuild.test.js +++ b/tests/unit/appRebuild.test.js @@ -9,7 +9,6 @@ describe('appRebuild tests', () => { let installer; let appRebuild; let bus; - let placements; const spec = { name: 'myapp', version: 8 }; beforeEach(() => { @@ -22,12 +21,10 @@ describe('appRebuild tests', () => { registerAppLocally: sinon.stub().resolves(InstallOutcome.INSTALLED), }; bus = { publish: sinon.stub() }; - placements = { removePlacement: sinon.stub().resolves() }; appRebuild = proxyquire('../../ZelBack/src/services/appLifecycle/appRebuild', { '../utils/fluxEventBus': bus, './appUninstaller': uninstaller, './appInstaller': installer, - '../appManagement/appPlacements': placements, '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, }); }); @@ -43,50 +40,6 @@ describe('appRebuild tests', () => { sinon.assert.notCalled(uninstaller.announceRemoval); }); - // A rebuild removes the app to install it here again, so the record of when - // this node took it stays - unless the app does not come back. - describe('the placement record', () => { - it('is kept through the removal and the reinstall', async () => { - await appRebuild.rebuildApp(spec); - - expect(uninstaller.removeAppLocally.firstCall.args[5]).to.deep.equal({ redeploying: true }); - sinon.assert.notCalled(placements.removePlacement); - }); - - it('goes when the app was removed and not reinstalled', async () => { - installer.registerAppLocally.resolves(InstallOutcome.FAILED); - - await appRebuild.rebuildApp(spec); - - sinon.assert.calledOnceWithExactly(placements.removePlacement, 'myapp'); - }); - - it('goes when the requirements are no longer met', async () => { - installer.checkAppRequirements.rejects(new Error('Insufficient RAM')); - - await appRebuild.rebuildApp(spec); - - sinon.assert.calledOnceWithExactly(placements.removePlacement, 'myapp'); - }); - - it('goes when the app was not installed and did not install', async () => { - uninstaller.removeAppLocally.resolves(RemovalOutcome.NOT_INSTALLED); - installer.registerAppLocally.resolves(InstallOutcome.FAILED); - - await appRebuild.rebuildApp(spec); - - sinon.assert.calledOnceWithExactly(placements.removePlacement, 'myapp'); - }); - - it('stays when the removal did not finish, for the removal that finishes it', async () => { - uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); - - await appRebuild.rebuildApp(spec); - - sinon.assert.notCalled(placements.removePlacement); - }); - }); - it('leaves the installer\'s teardown silent, so the network is told once, here', async () => { await appRebuild.rebuildApp(spec); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 988517054e..da1d8d9371 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -83,7 +83,10 @@ describe('appStartupManager tests', () => { sinon.stub(globalStateStub, 'waitForDaemonReady').resolves(); sinon.stub(globalStateStub, 'waitForBootContainerStateSettled').resolves(); - appPlacementsStub = { reconcilePlacements: sinon.stub().resolves({ created: 0, removed: 0 }) }; + appPlacementsStub = { + removeStrayPlacements: sinon.stub().resolves(0), + backfillPlacements: sinon.stub().resolves({ created: 0, fromLocationRows: 0 }), + }; appQueryServiceStub = { installedApps: sinon.stub().resolves({ status: 'success', data: [] }), @@ -942,30 +945,56 @@ describe('appStartupManager tests', () => { await appStartupManager.manageAppsOnBoot(KEEP); expect(logStub.error.calledWithMatch(/not opened in the firewall: helper crashed/)).to.equal(true); - // once for the placement records, twice for the reconcile - expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(3); + // once each for tidying and backfilling the placement records, twice for the reconcile + expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(4); }); // The first announcement reads these records, and it waits for boot to settle. - it('makes the placement records match the installed apps before it opens their ports', async () => { + it('gives the installed apps their records before it opens their ports', async () => { fluxNetworkHelperStub.getLocalSocketAddress.resolves('203.0.113.40:16127'); await appStartupManager.manageAppsOnBoot(KEEP); - sinon.assert.calledOnceWithExactly(appPlacementsStub.reconcilePlacements, ['web', 'db'], '203.0.113.40:16127'); - sinon.assert.callOrder(appPlacementsStub.reconcilePlacements, portManagerStub.appsWithPorts); + sinon.assert.calledOnceWithExactly(appPlacementsStub.backfillPlacements, ['web', 'db'], '203.0.113.40:16127'); + sinon.assert.callOrder(appPlacementsStub.backfillPlacements, portManagerStub.appsWithPorts); }); - it('logs records it could not reconcile and goes on booting', async () => { - appPlacementsStub.reconcilePlacements.rejects(new Error('mongo went away')); + it('logs records it could not backfill and goes on booting', async () => { + appPlacementsStub.backfillPlacements.rejects(new Error('mongo went away')); await appStartupManager.manageAppsOnBoot(KEEP); - expect(logStub.error.calledWithMatch(/placement records not reconciled: mongo went away/)).to.equal(true); + expect(logStub.error.calledWithMatch(/placement records not backfilled: mongo went away/)).to.equal(true); expect(logStub.info.calledWithMatch(/node confirmed, reconciling/)).to.equal(true); expect(globalStateStub.bootContainerStateSettled).to.equal(true); }); + it('removes stray records first, against the installed apps', async () => { + await appStartupManager.manageAppsOnBoot(KEEP); + + sinon.assert.calledOnceWithExactly(appPlacementsStub.removeStrayPlacements, ['web', 'db']); + sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, appPlacementsStub.backfillPlacements); + }); + + // A boot that stops before it reconciles the apps still clears stray records. + it('removes stray records on a boot that stops early, and backfills none', async () => { + fluxNetworkHelperStub.isNodeDos.returns(true); + + await appStartupManager.manageAppsOnBoot(KEEP); + + sinon.assert.calledOnce(appPlacementsStub.removeStrayPlacements); + sinon.assert.notCalled(appPlacementsStub.backfillPlacements); + }); + + it('logs stray records it could not remove and goes on booting', async () => { + appPlacementsStub.removeStrayPlacements.rejects(new Error('mongo went away')); + + await appStartupManager.manageAppsOnBoot(KEEP); + + expect(logStub.error.calledWithMatch(/stray placement records not removed: mongo went away/)).to.equal(true); + sinon.assert.calledOnce(appPlacementsStub.backfillPlacements); + }); + it('opens nothing when the apps are all being removed', async () => { appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'web' }] }); await appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 9317bb8e46..8b6c868db8 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -1029,10 +1029,10 @@ describe('appUninstaller tests', () => { sinon.assert.notCalled(runtimeStateStub.remove); }); - // When this node took an app is kept for as long as the app stays on it: a - // redeploy removes and reinstalls it without it ever leaving. + // The record follows the app's volume: a removal that deletes the volume + // deletes it, and one that keeps the volume keeps it. describe('the placement record', () => { - it('goes with an app that leaves this node', async () => { + it('goes with the app\'s volume', async () => { const uninstaller = buildUninstaller(composedSpec); const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, true); @@ -1041,15 +1041,6 @@ describe('appUninstaller tests', () => { sinon.assert.calledOnceWithExactly(appPlacementsStub.removePlacement, 'testapp'); }); - it('stays when the app is removed to be redeployed here', async () => { - const uninstaller = buildUninstaller(composedSpec); - - const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, false, { redeploying: true }); - - expect(outcome).to.equal(RemovalOutcome.REMOVED); - sinon.assert.notCalled(appPlacementsStub.removePlacement); - }); - it('stays when one component is removed and the app with it is not', async () => { const uninstaller = buildUninstaller(composedSpec); From c9eec5cbf1f863e69e69e710b6cd162ae29cb413 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:32:20 +0100 Subject: [PATCH 343/369] test(harness): 3502 a soft redeploy keeps the placement record and a hard one starts a new one; boot tidies and backfills Co-Authored-By: Claude Opus 5.5 --- test-infra/runner/framework/node-client.js | 4 +- ...s-its-own-record-of-when-it-took-an-app.js | 51 ++++++++++++++----- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index 9e26eee072..34af75674f 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -222,6 +222,7 @@ export function nodeClient(nodeNum) { 'app:installOutcome', 'app:removed', 'app:rebuilt', + 'app:softRedeployed', 'reinstall:passEnded', 'app:componentRedeployed', 'app:specStored', @@ -229,7 +230,8 @@ export function nodeClient(nodeNum) { // what this node records of when it took each app it holds 'app:placementRecorded', 'app:placementRemoved', - 'app:placementsReconciled', + 'app:placementsTidied', + 'app:placementsBackfilled', 'app:announcementRefused', 'app:announcing', 'fileoperation:imageAcquired', diff --git a/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js index 1199740cd0..fa147252db 100644 --- a/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js +++ b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js @@ -16,14 +16,15 @@ import { fluxTeamKey } from '../framework/keys.js'; import { authenticate } from '../auth.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; -// A node records when it took an app, once (app:placementRecorded), and announces -// that as the app's runningSince for as long as the app stays on it. The -// network's copy of the node's own location row is not where it comes from: that -// row lapses while the app goes on running, and a redeploy removes and -// reinstalls the app without it leaving. The record goes with the app -// (app:placementRemoved), and at boot the records are made to match what is -// installed (app:placementsReconciled), dating a missing one at the node's own -// row. +// A node records when it took an app (app:placementRecorded) and announces that +// as the app's runningSince. The record follows the app's data on the node: a +// hard install, which creates the app's volume, writes it; a removal, which +// deletes the volume, deletes it (app:placementRemoved); a soft redeploy keeps +// the volume and the record. The network's copy of the node's own location row +// is not where it comes from, because that row lapses while the app goes on +// running. At boot, records of apps not installed are removed +// (app:placementsTidied) and an installed app without one is dated at the +// node's own row (app:placementsBackfilled). const REAL_NODES = 3; const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); @@ -115,17 +116,37 @@ describe('a node keeps its own record of when it took an app', function () { expect(stored, 'the holder re-dated the app when its row lapsed').to.deep.equal(others.map(() => announcedSince)); }); - it('keeps the record, and the time it announces, through a hard redeploy', async function () { + it('keeps the record, and the time it announces, through a soft redeploy', async function () { + this.timeout(300000); + const afterIds = marks(); + const body = await env.clients[idx].redeployApp(appName, await teamAuth(), { force: false }); + const { data: redeployed } = await env.clients[idx].waitForEvent('app:softRedeployed', (d) => d.name === appName, 240000, { afterId: afterIds[idx] }); + expect(redeployed.outcome, `the redeploy did not reinstall the app: ${body.slice(-400)}`).to.equal('installed'); + + await assertNoEvent(env.clients[idx], 'app:placementRemoved', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'the soft redeploy re-dated the app').to.deep.equal(others.map(() => announcedSince)); + }); + + // A hard redeploy starts the app again from an empty volume, so it starts a + // new record: the copy holds none of the data the old date stood for. + it('starts a new record, and announces it, after a hard redeploy', async function () { this.timeout(300000); const afterIds = marks(); const body = await env.clients[idx].redeployApp(appName, await teamAuth(), { force: true }); const { data: rebuilt } = await env.clients[idx].waitForEvent('app:rebuilt', (d) => d.name === appName, 240000, { afterId: afterIds[idx] }); expect(rebuilt.outcome, `the redeploy did not reinstall the app: ${body.slice(-400)}`).to.equal('installed'); - await assertNoEvent(env.clients[idx], 'app:placementRemoved', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); - await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + await env.clients[idx].waitForEvent('app:placementRemoved', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); + expect(recorded.source).to.equal('install'); + const placedAt = new Date(recorded.placedAt).getTime(); + expect(placedAt, 'the new record is not newer than the old one').to.be.above(announcedSince); + const stored = await nextStoredRunningSince(marks()); - expect(stored, 'the redeploy re-dated the app').to.deep.equal(others.map(() => announcedSince)); + expect(stored, 'the announced runningSince is not the new record').to.deep.equal(others.map(() => placedAt)); + announcedSince = placedAt; }); // A node that took an app before it had these records gets one at boot, dated @@ -144,8 +165,10 @@ describe('a node keeps its own record of when it took an app', function () { await restartFluxos(env.clients[idx].container); await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); - const { data: reconciled } = await env.clients[idx].waitForEvent('app:placementsReconciled', () => true, 10000, { afterId: afterIds[idx] }); - expect(reconciled, 'the boot did not reconcile the records').to.deep.equal({ created: 1, fromLocationRows: 1, removed: 1 }); + const { data: tidied } = await env.clients[idx].waitForEvent('app:placementsTidied', () => true, 10000, { afterId: afterIds[idx] }); + expect(tidied, 'the boot did not remove the record of the app it does not hold').to.deep.equal({ removed: 1 }); + const { data: backfilled } = await env.clients[idx].waitForEvent('app:placementsBackfilled', () => true, 10000, { afterId: afterIds[idx] }); + expect(backfilled, 'the boot did not give the app a record').to.deep.equal({ created: 1, fromLocationRows: 1 }); const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); expect(recorded.source).to.equal('locationRow'); expect(new Date(recorded.placedAt).getTime(), 'the missing record was not dated at the node\'s own row').to.equal(rowSince); From 630dd584f1335492c21bde9ea7ed7a4087dff119 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:42:38 +0100 Subject: [PATCH 344/369] fix(apps): boot runs its placement steps again until they succeed Removing stray records and backfilling missing ones are database steps, and boot holds at them as it holds at its others, so the first announcement never goes out over installed apps without their records. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 18 ++++++------------ tests/unit/appStartupManager.test.js | 19 ++++++++++--------- 2 files changed, 16 insertions(+), 21 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index 6e273a9b0c..dd792ff034 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -398,33 +398,27 @@ async function removeAllApps(reason) { /** * Remove the placement records of apps not installed here, on every boot - * whichever way it goes on. A failure is logged and boot goes on: a stray record - * is never read, and the next boot tries again. + * whichever way it goes on. Run again until it succeeds. * @returns {Promise} */ async function tidyPlacementsOnBoot() { - try { + await untilSucceeds('Removing stray placement records', async () => { const installed = await getInstalledAppsFromDb(); await appPlacements.removeStrayPlacements(installed.map((app) => app.name)); - } catch (error) { - log.error(`appStartupManager - stray placement records not removed: ${error.message}`); - } + }); } /** * Give each installed app without a placement record one, before this node's - * first announcement reads them. A failure is logged and boot goes on: an app - * without a record is announced from its location row. + * first announcement reads them. Run again until it succeeds. * @returns {Promise} */ async function backfillPlacementsOnBoot() { - try { + await untilSucceeds('Backfilling placement records', async () => { const installed = await getInstalledAppsFromDb(); const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); await appPlacements.backfillPlacements(installed.map((app) => app.name), localSocketAddr); - } catch (error) { - log.error(`appStartupManager - placement records not backfilled: ${error.message}`); - } + }); } async function manageAppsOnBoot(bootContext) { diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index da1d8d9371..140b59e6cb 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -959,14 +959,15 @@ describe('appStartupManager tests', () => { sinon.assert.callOrder(appPlacementsStub.backfillPlacements, portManagerStub.appsWithPorts); }); - it('logs records it could not backfill and goes on booting', async () => { - appPlacementsStub.backfillPlacements.rejects(new Error('mongo went away')); + // Boot holds at a database step that fails, so the first announcement never + // goes out over apps without their records. + it('backfills the records again until it succeeds, before it goes on', async () => { + appPlacementsStub.backfillPlacements.onFirstCall().rejects(new Error('mongo went away')); await appStartupManager.manageAppsOnBoot(KEEP); - expect(logStub.error.calledWithMatch(/placement records not backfilled: mongo went away/)).to.equal(true); - expect(logStub.info.calledWithMatch(/node confirmed, reconciling/)).to.equal(true); - expect(globalStateStub.bootContainerStateSettled).to.equal(true); + sinon.assert.calledTwice(appPlacementsStub.backfillPlacements); + sinon.assert.callOrder(appPlacementsStub.backfillPlacements, portManagerStub.appsWithPorts); }); it('removes stray records first, against the installed apps', async () => { @@ -986,13 +987,13 @@ describe('appStartupManager tests', () => { sinon.assert.notCalled(appPlacementsStub.backfillPlacements); }); - it('logs stray records it could not remove and goes on booting', async () => { - appPlacementsStub.removeStrayPlacements.rejects(new Error('mongo went away')); + it('removes stray records again until it succeeds, before it goes on', async () => { + appPlacementsStub.removeStrayPlacements.onFirstCall().rejects(new Error('mongo went away')); await appStartupManager.manageAppsOnBoot(KEEP); - expect(logStub.error.calledWithMatch(/stray placement records not removed: mongo went away/)).to.equal(true); - sinon.assert.calledOnce(appPlacementsStub.backfillPlacements); + sinon.assert.calledTwice(appPlacementsStub.removeStrayPlacements); + sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, appPlacementsStub.backfillPlacements); }); it('opens nothing when the apps are all being removed', async () => { From 8c4d6f1982adf4898d2e602d785dfc232d63dbe7 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 09:44:00 +0100 Subject: [PATCH 345/369] test(harness): 3504 syncs a returning node against peers that hold only a refused announcement; 3503 asserts what can fail 3504's joiner already holds a node's row and readable announcement, and every peer serves a newer announcement for that node whose entries it refuses, the shape a peer that has not upgraded holds. 3503 asserts no placement record only where installapplocally could write one, and that a test install leaves nothing installed. Co-Authored-By: Claude Opus 5.5 --- ...03-who-may-place-or-test-an-app-by-hand.js | 34 +++++++------------ ...ouncement-is-judged-one-entry-at-a-time.js | 24 +++++++------ 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js index 97aade058b..115476721e 100644 --- a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js +++ b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js @@ -32,15 +32,16 @@ describe('who may place or test an app by hand', function () { const authAs = async (key) => (await authenticate(client().url, key)).zelidauth; const marks = () => env.clients.map((c) => c.getLastEventId()); - // Since `afterIds` the node recorded no placement of the app and announced it - // nowhere, and nothing of it is left installed. - async function assertNothingHeld(afterIds) { + // installapplocally placed nothing: no record of taking the app was written, + // and the app is not installed. + async function assertNotPlaced(afterIds) { await assertNoEvent(client(), 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[NODE] }); - await assertNoEvent(client(), 'app:running', (d) => d.apps?.some((a) => a.name === appName), 0, { afterId: afterIds[NODE] }); - await Promise.all(env.clients.map((c, i) => assertNoEvent(c, 'network:apprunning', - (d) => d.apps?.some((a) => a.name === appName), 0, { afterId: afterIds[i] }))); - expect(await dbClient(NODE + 1).localApp(appName), 'the app is left installed on the node').to.equal(null); + expect(await dbClient(NODE + 1).localApp(appName), 'the app is installed on the node').to.equal(null); } + // A test install leaves nothing of the app behind. + const assertTornDown = () => waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { + timeout: 60000, interval: 2000, label: 'the test install is torn down', + }); before(async function () { this.timeout(420000); @@ -79,7 +80,7 @@ describe('who may place or test an app by hand', function () { const body = await client().installAppLocally(messageHash, await authAs(appOwnerKey())); expect(body, 'the owner was not refused').to.match(/Unauthorized/); - await assertNothingHeld(afterIds); + await assertNotPlaced(afterIds); }); it('does not place it for the flux team either', async function () { @@ -88,39 +89,30 @@ describe('who may place or test an app by hand', function () { const body = await client().installAppLocally(messageHash, await authAs(fluxTeamKey())); expect(body, 'installapplocally read the temporary message').to.match(new RegExp(`Application Specifications of ${messageHash} not found`)); - await assertNothingHeld(afterIds); + await assertNotPlaced(afterIds); }); it('refuses another user testing it', async function () { this.timeout(120000); - const afterIds = marks(); const body = await client().testAppInstall(messageHash, await authAs(userKey())); expect(body, 'another user was not refused').to.match(/Unauthorized/); - await assertNothingHeld(afterIds); + await assertTornDown(); }); it('lets its owner test it, and leaves nothing behind', async function () { this.timeout(300000); - const afterIds = marks(); const body = await client().testAppInstall(messageHash, await authAs(appOwnerKey())); expect(body, 'the owner\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); - await waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { - timeout: 60000, interval: 2000, label: 'the test install is torn down', - }); - await assertNothingHeld(afterIds); + await assertTornDown(); }); it('lets the flux team test it', async function () { this.timeout(300000); - const afterIds = marks(); const body = await client().testAppInstall(messageHash, await authAs(fluxTeamKey())); expect(body, 'the flux team\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); - await waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { - timeout: 60000, interval: 2000, label: 'the test install is torn down', - }); - await assertNothingHeld(afterIds); + await assertTornDown(); }); }); diff --git a/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js index fd61c32fea..5f63653ed5 100644 --- a/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js +++ b/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js @@ -17,9 +17,10 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // // A peer that has not upgraded serves whatever it last recorded for a node, // which may be an announcement whose every entry the joiner refuses. The joiner -// prunes a node's rows against the newest announcement it can store, so such a -// message never deletes the rows an older readable one stood behind, and it -// never takes that one's place in the joiner's own log. +// here is a node coming back: it already holds a row for that node and the +// readable announcement behind it. It prunes a node's rows only against an +// announcement it can store, so the refused one deletes neither the row nor the +// readable announcement in the log it serves its own peers. const RUNNING = [0, 1, 2, 3]; const JOINER = 4; @@ -61,18 +62,21 @@ describe('a synced announcement is judged one entry at a time', function () { await waitForNodeStatus(joiner, (d) => d.confirmed === true, 30000); const at = Date.now(); + // What the joiner already holds about REFUSED_NODE from before it went away: + // the readable announcement and the row it wrote. + await dbClient(JOINER + 1).seedAppStateEvent(await apprunningEvent({ nodeNum: REFUSED_NODE, broadcastedAt: at - 5000, apps: [held] })); + await dbClient(JOINER + 1).seedAppLocation({ + name: held, ip: socketAddr(REFUSED_NODE), hash: `hash-${held}`, broadcastedAt: at - 5000, runningSince: at - 5000, + }); + // What every peer serves: for MIXED_NODE one readable entry and one without a + // hash; for REFUSED_NODE a newer announcement whose only entry has no hash, + // the one a peer that has not upgraded holds for a node it last heard that from. const events = [ - // One readable entry and one without a hash. await apprunningEvent({ nodeNum: MIXED_NODE, broadcastedAt: at, apps: [kept, { name: dropped, runningSince: new Date(at).toISOString() }], }), - // A readable announcement, then a newer one whose only entry has no hash: - // what a peer that has not upgraded holds for a node it last heard that from. - await apprunningEvent({ - nodeNum: REFUSED_NODE, broadcastedAt: at - 5000, apps: [held], dedupKey: 'v2:readable', - }), await apprunningEvent({ nodeNum: REFUSED_NODE, broadcastedAt: at, apps: [{ name: probe, runningSince: new Date(at).toISOString() }], }), @@ -102,7 +106,7 @@ describe('a synced announcement is judged one entry at a time', function () { expect(refused.some((r) => r.ip === socketAddr(MIXED_NODE) && r.names.includes(dropped)), 'the joiner did not report the entry it refused').to.equal(true); }); - it('keeps a node\'s rows past a newer announcement whose every entry it refuses', async function () { + it('keeps the row it holds for a node past a newer announcement whose every entry it refuses', async function () { const names = await rowNames(REFUSED_NODE); expect(names, 'the refused announcement pruned the rows the readable one stood behind').to.include(held); expect(names, 'the joiner stored an entry without a hash').to.not.include(probe); From e0a3af2dd49f9cdfdae3c0389b7b51cc6cfc0dd8 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 10:34:40 +0100 Subject: [PATCH 346/369] docs(apps): a hard install's placement record write must succeed Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appInstaller.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 73b3f490a1..4163f6741e 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -549,7 +549,9 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f folderStates.installed(appSpecifications); claimed = true; // The app's volumes are created next: this node's record of when it took - // the app starts now. + // the app starts now. The write must succeed, like the insert above: an + // install that went on without it would leave any earlier record of the + // app standing, dated before this volume existed. if (!test) await appPlacements.recordPlacement(appSpecifications.name); const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; From 9ed3db3986952b8f0fe85b12684930c1c2b5b7dd Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 11:03:47 +0100 Subject: [PATCH 347/369] test(harness): 2237 holds the crashed node's confirmation until the bystander holds its move Co-Authored-By: Claude Opus 5.5 --- ...or-a-crash-interrupted-is-finished-once.js | 32 +++++++++++++++---- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js b/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js index ac5f0f6b9b..d489bb68a0 100644 --- a/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js +++ b/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js @@ -8,7 +8,7 @@ import { configuredIpOf, crashFluxos, moveNodeAddress, releaseFluxos, } from '../framework/container.js'; import { - clearNodeAddress, clearRpcFailures, failRpc, getJournal, setNodeAddress, + clearNodeAddress, clearRpcFailures, failRpc, getJournal, holdRpc, releaseRpc, setNodeAddress, } from '../framework/daemon-control.js'; import { waitFor } from '../framework/wait.js'; import { bootAndPeer } from '../framework/reconciler-suite.js'; @@ -25,6 +25,10 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // Node 0 has its confirmation refused once; node 1 is crashed between // recording its move and acting on it. Node 2 is a bystander that holds the // events and reads the list. +// +// Node 1 runs no app, so a peer that hears its move once the chain lists it at +// the new address only has nothing to prove the move by, and drops it. Its +// confirmation is held after it is back until the bystander holds the move. const REFUSED = 0; const CRASHED = 1; @@ -39,6 +43,7 @@ describe('a move a failure or a crash interrupted is finished once', function () dumpLogsOnFailure(() => env); const moved = []; const failing = []; + const holding = []; const client = (i) => env.clients[i]; const socketOf = (ip) => (String(ip).includes(':') ? String(ip) : `${ip}:${API_PORT}`); const nodeList = async () => (await client(BYSTANDER).get(`/daemon/viewdeterministiczelnodelist?_=${Date.now()}`)).data || []; @@ -59,6 +64,17 @@ describe('a move a failure or a crash interrupted is finished once', function () return { from: socketOf(from), to: socketOf(to) }; } + // The move the bystander holds from node i's old address. + async function bystanderHolds(i, { from, to }) { + let held; + await waitFor(async () => { + held = (await dbClient(BYSTANDER + 1).getAppStateEvents({ ip: from, type: 'ipchanged' })) + .find((event) => event.data?.newIP === to); + return Boolean(held); + }, { timeout: 180000, interval: 5000, label: `the bystander holds node ${i}'s move` }); + return held; + } + // The move as the node recorded it, the move the bystander holds from its old // address, and where the list has it. async function expectFinishedOnce(i, { from, to }, pubkey) { @@ -68,12 +84,7 @@ describe('a move a failure or a crash interrupted is finished once', function () const record = await dbClient(i + 1).ownAddressRecord(); expect(record.socketAddress, `node ${i}'s recorded address`).to.equal(to); expect(record.recentIpChanges, `node ${i}'s IP changes counted`).to.have.lengthOf(1); - let held; - await waitFor(async () => { - held = (await dbClient(BYSTANDER + 1).getAppStateEvents({ ip: from, type: 'ipchanged' })) - .find((event) => event.data?.newIP === to); - return Boolean(held); - }, { timeout: 120000, interval: 5000, label: `the bystander holds node ${i}'s move` }); + const held = await bystanderHolds(i, { from, to }); expect(held.data.broadcastedAt, `the time node ${i}'s move is told with`).to.equal(record.movedAt); expect(await configuredIpOf(client(i).container), `node ${i}'s configured IP`).to.equal(to.split(':')[0]); } @@ -93,6 +104,7 @@ describe('a move a failure or a crash interrupted is finished once', function () this.timeout(60000); await Promise.all(moved.map((ip) => clearNodeAddress(ip).catch(() => {}))); await Promise.all(failing.map((ip) => clearRpcFailures(ip).catch(() => {}))); + await Promise.all(holding.map((ip) => releaseRpc(ip).catch(() => {}))); await env?.teardown(); }); @@ -125,8 +137,14 @@ describe('a move a failure or a crash interrupted is finished once', function () await node.waitForEvent('checkpoint:held', (d) => d.name === BEFORE_RECONCILE && d.key === addresses.to, DETECTION_MS, { afterId: anchor }); expect((await dbClient(CRASHED + 1).ownAddressRecord()).socketAddress, 'fixture: the move recorded before the crash').to.equal(addresses.to); await crashFluxos(node.container, { hold: true }); + // The daemon knows a node by the address its calls come from. + holding.push(toIp); + await holdRpc(toIp, 'createconfirmationtransaction'); await releaseFluxos(node.container); + await bystanderHolds(CRASHED, addresses); + expect(await listedAt(pubkey), 'fixture: the bystander heard the move with the chain listing the old address').to.equal(addresses.from); + await releaseRpc(toIp); await expectFinishedOnce(CRASHED, addresses, pubkey); }); }); From 7da4a76ea7873f3cfc2ebcfec1cee7d5031fd6c8 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 11:10:48 +0100 Subject: [PATCH 348/369] test(harness): 2237 has the restarted node dial the bystander it tells its move to Co-Authored-By: Claude Opus 5.5 --- ...ilure-or-a-crash-interrupted-is-finished-once.js | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js b/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js index d489bb68a0..eb5bd892c4 100644 --- a/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js +++ b/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js @@ -10,6 +10,7 @@ import { import { clearNodeAddress, clearRpcFailures, failRpc, getJournal, holdRpc, releaseRpc, setNodeAddress, } from '../framework/daemon-control.js'; +import { askToBeDialed } from '../framework/peer-topology.js'; import { waitFor } from '../framework/wait.js'; import { bootAndPeer } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; @@ -27,8 +28,9 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // events and reads the list. // // Node 1 runs no app, so a peer that hears its move once the chain lists it at -// the new address only has nothing to prove the move by, and drops it. Its -// confirmation is held after it is back until the bystander holds the move. +// the new address only has nothing to prove the move by, and drops it. Once it +// is back its confirmation is held, and it is asked to dial the bystander, until +// the bystander holds the move. const REFUSED = 0; const CRASHED = 1; @@ -140,7 +142,14 @@ describe('a move a failure or a crash interrupted is finished once', function () // The daemon knows a node by the address its calls come from. holding.push(toIp); await holdRpc(toIp, 'createconfirmationtransaction'); + const bystander = client(BYSTANDER); + const peeredAnchor = bystander.getLastEventId(); await releaseFluxos(node.container); + // A node tells its move to each peer as it connects. + await waitFor(async () => askToBeDialed(bystander, node, API_PORT).then(() => true, () => false), { + timeout: 180000, interval: 5000, label: `node ${CRASHED} takes the bystander's ask to be dialed`, + }); + await bystander.waitForEvent('peers:added', (d) => d.ip === toIp, 120000, { afterId: peeredAnchor }); await bystanderHolds(CRASHED, addresses); expect(await listedAt(pubkey), 'fixture: the bystander heard the move with the chain listing the old address').to.equal(addresses.from); From b77a6442f4cd31dca1edf1c3ac4348f58f05ab2f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:01:57 +0100 Subject: [PATCH 349/369] docs(apps): name when a placement record outlives its app Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appManagement/appPlacements.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js index 6d7eeceb9c..b4b60e0c1d 100644 --- a/ZelBack/src/services/appManagement/appPlacements.js +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -12,9 +12,12 @@ const fluxEventBus = require('../utils/fluxEventBus'); // // The announcer reads runningSince from here and nowhere else, because the // network's copy of this node's own location rows lapses in normal running - -// expiry, eviction, sigterm - while the app goes on running. A record outliving -// its app is never read: only an installed app's record is, and a hard install -// overwrites whatever it finds. +// expiry, eviction, sigterm - while the app goes on running. +// +// A record outlives its app only when a soft redeploy removes the app and its +// reinstall does not happen: the volume is left on disk and the record with it, +// until boot removes records of apps not installed. Only an installed app's record +// is read, and the app returns only through a hard install, which overwrites it. // // Kept apart from the app's specification in appsInformation: a spec update // deletes and re-inserts that, and this record must outlive it. From 23ecdd8fb4058ac909e612d0eaae6a2131c0a495 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:30:43 +0100 Subject: [PATCH 350/369] fix(network): a node without an address asks benchmark for one every 15 s, 48 times before it is out of service Co-Authored-By: Claude Opus 5.5 --- ZelBack/config/default.js | 7 ++++--- tests/unit/globalconfig/default.js | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index 42d408dff4..549f7fa7f6 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -481,9 +481,10 @@ module.exports = { peerShortfallDosPerMinute: 0.13, // While this node knows no address of its own it asks benchmark to detect its // public IP this often, and is out of service after this many asks find none. - // Benchmark caches its answer for 69 s. - publicIpAskIntervalMs: 90 * 1000, - publicIpAsksBeforeDos: 8, + // Benchmark caches an address it found for 69 s and none it failed to find, so + // each ask while none is known is a fresh lookup against public IP services. + publicIpAskIntervalMs: 15 * 1000, + publicIpAsksBeforeDos: 48, // The apps port check. timeoutsMs: how long it waits before testing again // after a pass, an error, a failed port, once its count reaches the DOS // threshold, and after an app error. thresholds: once portsHighEdge ports diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 561c4890f4..d366672e44 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -266,8 +266,8 @@ module.exports = { availabilityCheckIntervalMs: 300000, availabilityRecheckMs: 60000, peerShortfallDosPerMinute: 0.13, - publicIpAskIntervalMs: 90000, - publicIpAsksBeforeDos: 8, + publicIpAskIntervalMs: 15000, + publicIpAsksBeforeDos: 48, appsPortCheck: { timeoutsMs: { default: 3600000, error: 60000, failure: 15000, dos: 300000, appError: 240000, From b4eb78cf6ed58ee3e3a88445ef53657782747974 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:30:43 +0100 Subject: [PATCH 351/369] fix(apps): the announcer waits for this node's address and for every installed app's placement record Placement records are backfilled once per start by a migration that waits for the node's address, which its location rows are filed under. Every boot path backfills, and the first announcement goes out as soon as the address is known and the backfill has written. Co-Authored-By: Claude Opus 5.5 --- .../appLifecycle/appStartupManager.js | 14 ------ .../services/appManagement/appPlacements.js | 35 +++++++++---- .../services/appMessaging/peerNotification.js | 12 ++--- ZelBack/src/services/fluxNetworkHelper.js | 22 +++++++++ ZelBack/src/services/migrations/placements.js | 27 ++++++++++ ZelBack/src/services/serviceManager.js | 4 ++ tests/unit/appPlacements.test.js | 27 ++++++---- tests/unit/appStartupManager.test.js | 33 ++----------- tests/unit/fluxNetworkHelper.test.js | 49 +++++++++++++++++++ tests/unit/peerNotification.test.js | 40 ++++++++++++++- tests/unit/placementsMigration.test.js | 41 ++++++++++++++++ 11 files changed, 233 insertions(+), 71 deletions(-) create mode 100644 ZelBack/src/services/migrations/placements.js create mode 100644 tests/unit/placementsMigration.test.js diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index dd792ff034..81916d8979 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -408,19 +408,6 @@ async function tidyPlacementsOnBoot() { }); } -/** - * Give each installed app without a placement record one, before this node's - * first announcement reads them. Run again until it succeeds. - * @returns {Promise} - */ -async function backfillPlacementsOnBoot() { - await untilSucceeds('Backfilling placement records', async () => { - const installed = await getInstalledAppsFromDb(); - const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - await appPlacements.backfillPlacements(installed.map((app) => app.name), localSocketAddr); - }); -} - async function manageAppsOnBoot(bootContext) { try { await tidyPlacementsOnBoot(); @@ -484,7 +471,6 @@ async function manageAppsOnBoot(bootContext) { } log.info('appStartupManager - Daemon, DB, and node confirmed, reconciling apps'); - await backfillPlacementsOnBoot(); await openInstalledAppPorts(); await reconcileAppsOnBoot(); } finally { diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js index b4b60e0c1d..60130311e3 100644 --- a/ZelBack/src/services/appManagement/appPlacements.js +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -2,6 +2,7 @@ const config = require('config'); const log = require('../../lib/log'); const dbHelper = require('../dbHelper'); const fluxEventBus = require('../utils/fluxEventBus'); +const { AsyncGate } = require('../utils/asyncGate'); // When this node took each app it holds: one record per app, keyed by name. // @@ -14,10 +15,11 @@ const fluxEventBus = require('../utils/fluxEventBus'); // network's copy of this node's own location rows lapses in normal running - // expiry, eviction, sigterm - while the app goes on running. // -// A record outlives its app only when a soft redeploy removes the app and its -// reinstall does not happen: the volume is left on disk and the record with it, -// until boot removes records of apps not installed. Only an installed app's record -// is read, and the app returns only through a hard install, which overwrites it. +// A record outlives its app when a soft redeploy removes the app and its reinstall +// does not happen, which leaves the volume on disk and the record with it, or when +// the app is removed while the backfill below is writing. Boot removes records of +// apps not installed. Only an installed app's record is read, and the app returns +// only through a hard install, which overwrites it. // // Kept apart from the app's specification in appsInformation: a spec update // deletes and re-inserts that, and this record must outlive it. @@ -26,6 +28,9 @@ const appsLocalDatabase = config.database.appslocal.database; const { appsPlacements } = config.database.appslocal.collections; const { appsLocations } = config.database.appsglobal.collections; +// Open once every app installed at start has a record. +const backfilledGate = new AsyncGate(); + function localDatabase() { return dbHelper.databaseConnection().db(appsLocalDatabase); } @@ -97,19 +102,19 @@ async function removeStrayPlacements(installedNames) { } /** - * Give each installed app without a record one, before this node first announces - * it. It is dated at the runningSince this node's own location row holds, so the - * app keeps the place among its instances the network already gives it, and - * dated now when there is no row. A record that exists is left as it is. + * Give each installed app without a record one. It is dated at the runningSince + * this node's own location row holds, so the app keeps the place among its + * instances the network already gives it, and dated now when there is no row. A + * record that exists is left as it is. Opens waitForBackfill. * @param {Array} installedNames Names of the apps installed here. - * @param {string} localSocketAddr This node's address. + * @param {string} localSocketAddr This node's address, which its rows are filed under. * @returns {Promise<{created: number, fromLocationRows: number}>} What was written. */ async function backfillPlacements(installedNames, localSocketAddr) { const recorded = await placementsOf(installedNames); const missing = installedNames.filter((name) => !recorded.has(name)); - const ownRows = missing.length && localSocketAddr + const ownRows = missing.length ? await dbHelper.findInDatabase( dbHelper.databaseConnection().db(config.database.appsglobal.database), appsLocations, @@ -139,9 +144,18 @@ async function backfillPlacements(installedNames, localSocketAddr) { log.info(`appPlacements - recorded ${missing.length} placement(s), ${rowSince.size} from this node's location rows`); } fluxEventBus.publish('app:placementsBackfilled', { created: missing.length, fromLocationRows: rowSince.size }); + backfilledGate.open(); return { created: missing.length, fromLocationRows: rowSince.size }; } +/** + * Resolves once every app installed at start has a record. + * @returns {Promise} + */ +function waitForBackfill() { + return backfilledGate.wait(); +} + module.exports = { prepareCollection, recordPlacement, @@ -149,4 +163,5 @@ module.exports = { removePlacement, removeStrayPlacements, backfillPlacements, + waitForBackfill, }; diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index 545898602e..9c7f43f0e2 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -153,11 +153,10 @@ async function checkAndNotifyPeersOfRunningApps() { // best-effort. Resolves immediately in steady state; capped reconciler-side, // so a wedged reconcile cannot block the node's network presence. await appReconciler.waitForBootDrainSettled(); + // Every app installed at start has a record before the first snapshot reads them. + await appPlacements.waitForBackfill(); - const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - if (!localSocketAddr) { - throw new Error('Unable to detect Flux IP address'); - } + const localSocketAddr = await fluxNetworkHelper.waitForLocalSocketAddress(); // Stamped where the snapshot is taken, not where the message is built: every // consumer reads broadcastedAt as the time these apps were installed here, and @@ -211,9 +210,8 @@ async function checkAndNotifyPeersOfRunningApps() { try { // When this node took each app comes from its own record of that, never from // the network's copy of its location rows: those lapse in normal running - // while the app goes on running here. An app with no record - the boot step - // writes one for every installed app - is announced from its row, and from - // now without one. + // while the app goes on running here. An app with no record is announced + // from its row, and from now without one. const placements = await appPlacements.placementsOf(announceable.map((application) => application.name)); const unrecorded = []; // eslint-disable-next-line no-restricted-syntax diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 923da2d058..21427ae174 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -148,6 +148,7 @@ let storedFluxBenchAllowed = null; let tooManyIpChangesAt = null; let maxNumberOfIpChanges = 0; const { lruRateLimit } = require('./utils/rateLimit'); +const { AsyncGate } = require('./utils/asyncGate'); const { Privilege, authOf } = require('./utils/privileges'); // This node's socket address (ip:port) from benchmark @@ -171,6 +172,8 @@ let lastKnownSocketAddress = null; // Emits 'change' with the new address when the node learns one that differs // from lastKnownSocketAddress. const localSocketAddressEvents = new EventEmitter(); +// Open while localSocketAddress holds an address. +const localSocketAddressGate = new AsyncGate(); /** * The device an os.networkInterfaces() entry is bound on. A labelled address @@ -991,6 +994,8 @@ function setLocalSocketAddress(value) { // (Re)set the freshness deadline whenever the value changes, so getLocalSocketAddress // serves it without a benchmark RPC until it goes stale (cleared when set to null). localSocketAddressFreshUntil = localSocketAddress ? process.hrtime.bigint() + LOCAL_SOCKET_ADDRESS_TTL_NS : 0n; + if (localSocketAddress) localSocketAddressGate.open(); + else localSocketAddressGate.close(); // Told here because this is the one place the node learns what it is. The // peer manager needs it to keep this node out of its own peer draws - a node // that syncs from itself learns nothing, and it spends one of very few @@ -1010,6 +1015,22 @@ function getKnownLocalSocketAddress() { return lastKnownSocketAddress; } +/** + * This node's socket address, once it has one. Resolves as soon as benchmark + * reports an address to any caller of getLocalSocketAddress; while none is known + * the own-standing watch keeps asking for one. + * @returns {Promise} ip:port + */ +async function waitForLocalSocketAddress() { + let address = await getLocalSocketAddress(); + while (!address) { + // eslint-disable-next-line no-await-in-loop + await localSocketAddressGate.wait(); + address = localSocketAddress; + } + return address; +} + /** * @param {function(string): void} listener Called with the new address (ip:port) * each time the node learns an address that differs from the last one. @@ -3728,6 +3749,7 @@ module.exports = { checkFluxAvailability, getLocalSocketAddress, getKnownLocalSocketAddress, + waitForLocalSocketAddress, onLocalSocketAddressChange, offLocalSocketAddressChange, getFluxNodePrivateKey, diff --git a/ZelBack/src/services/migrations/placements.js b/ZelBack/src/services/migrations/placements.js new file mode 100644 index 0000000000..9eff23ab01 --- /dev/null +++ b/ZelBack/src/services/migrations/placements.js @@ -0,0 +1,27 @@ +const config = require('config'); +const dbHelper = require('../dbHelper'); +const fluxNetworkHelper = require('../fluxNetworkHelper'); +const appPlacements = require('../appManagement/appPlacements'); +const { localAppsInformation } = require('../utils/appConstants'); +const { untilSucceeds } = require('../utils/untilSucceeds'); + +/** + * Gives each app installed here without a placement record one, from this node's + * own location rows, once this node knows the address they are filed under. Run + * again until it succeeds. + * @returns {Promise} + */ +async function backfillPlacements() { + await untilSucceeds('Backfilling placement records', async () => { + const localSocketAddr = await fluxNetworkHelper.waitForLocalSocketAddress(); + const installed = await dbHelper.findInDatabase( + dbHelper.databaseConnection().db(config.database.appslocal.database), + localAppsInformation, + {}, + { projection: { _id: 0, name: 1 } }, + ); + await appPlacements.backfillPlacements(installed.map((app) => app.name), localSocketAddr); + }); +} + +module.exports = { backfillPlacements }; diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index 9dc4ae072c..4967afa496 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -50,6 +50,7 @@ const appQueryService = require('./appQuery/appQueryService'); const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); const daemonServiceUtils = require('./daemonService/daemonServiceUtils'); const paymentPositions = require('./migrations/paymentPositions'); +const placements = require('./migrations/placements'); const fluxService = require('./fluxService'); const geolocationService = require('./geolocationService'); const ipLocationSync = require('./appPlacement/ipLocationSync'); @@ -393,6 +394,9 @@ async function startFluxFunctions() { appStartupManager.manageAppsOnBoot(bootContext).catch((error) => { log.error(`App startup manager error: ${error.message}`); }); + // Not awaited: it waits for this node's address, and the announcer, the one + // reader of what it writes, waits for it. + placements.backfillPlacements(); // Above the daemon wait, because syncthing has nothing to do with the // daemon: the sentinel talks to syncthing's own API and reads userconfig, diff --git a/tests/unit/appPlacements.test.js b/tests/unit/appPlacements.test.js index eb07f1c903..42e1ee813c 100644 --- a/tests/unit/appPlacements.test.js +++ b/tests/unit/appPlacements.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai'); const config = require('config'); +const proxyquire = require('proxyquire'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const appPlacements = require('../../ZelBack/src/services/appManagement/appPlacements'); const { requireMongo } = require('./dbTestHelper'); @@ -151,16 +152,6 @@ describe('appPlacements', () => { expect(placedAt.getTime()).to.be.within(before, Date.now()); }); - it('dates an app now when this node does not know its own address', async () => { - await locations.insertOne(row('web', me, june)); - const before = Date.now(); - - await appPlacements.backfillPlacements(['web'], null); - - const [{ placedAt }] = await recorded(); - expect(placedAt.getTime()).to.be.within(before, Date.now()); - }); - it('leaves a record it already has, whatever the row says', async () => { await seed('web', june); await locations.insertOne(row('web', me, july)); @@ -179,4 +170,20 @@ describe('appPlacements', () => { expect((await recorded()).map((r) => r.name)).to.deep.equal(['gone', 'web']); }); }); + + describe('waitForBackfill', () => { + it('resolves once the backfill has written, and not before', async () => { + const fresh = proxyquire('../../ZelBack/src/services/appManagement/appPlacements', {}); + let resolved = false; + const waiting = fresh.waitForBackfill().then(() => { resolved = true; }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(resolved, 'resolved before the backfill ran').to.equal(false); + + await fresh.backfillPlacements(['web'], me); + await waiting; + + expect(resolved).to.equal(true); + expect((await recorded()).map((r) => r.name)).to.deep.equal(['web']); + }); + }); }); diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index 140b59e6cb..519c146711 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -85,7 +85,6 @@ describe('appStartupManager tests', () => { appPlacementsStub = { removeStrayPlacements: sinon.stub().resolves(0), - backfillPlacements: sinon.stub().resolves({ created: 0, fromLocationRows: 0 }), }; appQueryServiceStub = { @@ -945,46 +944,24 @@ describe('appStartupManager tests', () => { await appStartupManager.manageAppsOnBoot(KEEP); expect(logStub.error.calledWithMatch(/not opened in the firewall: helper crashed/)).to.equal(true); - // once each for tidying and backfilling the placement records, twice for the reconcile - expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(4); - }); - - // The first announcement reads these records, and it waits for boot to settle. - it('gives the installed apps their records before it opens their ports', async () => { - fluxNetworkHelperStub.getLocalSocketAddress.resolves('203.0.113.40:16127'); - - await appStartupManager.manageAppsOnBoot(KEEP); - - sinon.assert.calledOnceWithExactly(appPlacementsStub.backfillPlacements, ['web', 'db'], '203.0.113.40:16127'); - sinon.assert.callOrder(appPlacementsStub.backfillPlacements, portManagerStub.appsWithPorts); - }); - - // Boot holds at a database step that fails, so the first announcement never - // goes out over apps without their records. - it('backfills the records again until it succeeds, before it goes on', async () => { - appPlacementsStub.backfillPlacements.onFirstCall().rejects(new Error('mongo went away')); - - await appStartupManager.manageAppsOnBoot(KEEP); - - sinon.assert.calledTwice(appPlacementsStub.backfillPlacements); - sinon.assert.callOrder(appPlacementsStub.backfillPlacements, portManagerStub.appsWithPorts); + // once for tidying the placement records, twice for the reconcile + expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(3); }); it('removes stray records first, against the installed apps', async () => { await appStartupManager.manageAppsOnBoot(KEEP); sinon.assert.calledOnceWithExactly(appPlacementsStub.removeStrayPlacements, ['web', 'db']); - sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, appPlacementsStub.backfillPlacements); + sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, portManagerStub.appsWithPorts); }); // A boot that stops before it reconciles the apps still clears stray records. - it('removes stray records on a boot that stops early, and backfills none', async () => { + it('removes stray records on a boot that stops early', async () => { fluxNetworkHelperStub.isNodeDos.returns(true); await appStartupManager.manageAppsOnBoot(KEEP); sinon.assert.calledOnce(appPlacementsStub.removeStrayPlacements); - sinon.assert.notCalled(appPlacementsStub.backfillPlacements); }); it('removes stray records again until it succeeds, before it goes on', async () => { @@ -993,7 +970,7 @@ describe('appStartupManager tests', () => { await appStartupManager.manageAppsOnBoot(KEEP); sinon.assert.calledTwice(appPlacementsStub.removeStrayPlacements); - sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, appPlacementsStub.backfillPlacements); + sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, portManagerStub.appsWithPorts); }); it('opens nothing when the apps are all being removed', async () => { diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 2478ad4375..abdde5a462 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -404,6 +404,55 @@ describe('fluxNetworkHelper tests', () => { }); }); + describe('waitForLocalSocketAddress tests', () => { + let benchStub; + + beforeEach(() => { + benchStub = sinon.stub(benchmarkService, 'getBenchmarks'); + fluxNetworkHelper.setLocalSocketAddress(null); + }); + + afterEach(() => { + benchStub.restore(); + fluxNetworkHelper.setLocalSocketAddress(null); + }); + + it('answers at once when benchmark reports an address', async () => { + benchStub.resolves({ status: 'success', data: { ipaddress: '203.0.113.40:16127' } }); + + expect(await fluxNetworkHelper.waitForLocalSocketAddress()).to.equal('203.0.113.40:16127'); + }); + + it('waits while no address is known, and answers with the first one learned', async () => { + benchStub.resolves({ status: 'error' }); + let answer = null; + const waiting = fluxNetworkHelper.waitForLocalSocketAddress().then((address) => { answer = address; }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(answer, 'answered before an address was known').to.equal(null); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.41:16127'); + await waiting; + + expect(answer).to.equal('203.0.113.41:16127'); + }); + + it('waits again when the address it was woken for is gone before it reads it', async () => { + benchStub.resolves({ status: 'error' }); + let answer = null; + const waiting = fluxNetworkHelper.waitForLocalSocketAddress().then((address) => { answer = address; }); + await new Promise((resolve) => { setImmediate(resolve); }); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.42:16127'); + fluxNetworkHelper.setLocalSocketAddress(null); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(answer, 'answered with no address').to.equal(null); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.43:16127'); + await waiting; + expect(answer).to.equal('203.0.113.43:16127'); + }); + }); + describe('isFluxAvailable tests', () => { let stub; const ip = '127.0.0.1'; diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index 5aa39c84a1..6fdf2f3cd4 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -21,6 +21,8 @@ describe('peerNotification tests', () => { let announceCycle; let findOneInDatabaseStub; let placementsOfStub; + let waitForBackfillStub; + let waitForLocalSocketAddressStub; // One stub map, so a test that needs a different interval, expiry or cycle // length states only that difference instead of restating sixty lines. @@ -57,7 +59,7 @@ describe('peerNotification tests', () => { nodeTier: sinon.stub().resolves('cumulus'), }, '../fluxNetworkHelper': { - getLocalSocketAddress: sinon.stub().resolves('192.168.1.1:16127'), + waitForLocalSocketAddress: waitForLocalSocketAddressStub, }, '../geolocationService': { isStaticIP: sinon.stub().returns(true), @@ -116,7 +118,7 @@ describe('peerNotification tests', () => { runningAppsCache: new Set(), }, '../utils/nodeSigner': { nodeSigner: nodeSignerStub }, - '../appManagement/appPlacements': { placementsOf: placementsOfStub }, + '../appManagement/appPlacements': { placementsOf: placementsOfStub, waitForBackfill: waitForBackfillStub }, '../../lib/log': logStub, }); @@ -132,6 +134,8 @@ describe('peerNotification tests', () => { placementsOfStub = sinon.stub().callsFake(async (names) => new Map(names.map((name) => [name, new Date('2026-09-01T00:00:00.000Z')]))); enqueueAllStub = sinon.stub().resolves(); waitForBootDrainSettledStub = sinon.stub().resolves(); + waitForBackfillStub = sinon.stub().resolves(); + waitForLocalSocketAddressStub = sinon.stub().resolves('192.168.1.1:16127'); storeAppRunningMessageStub = sinon.stub().resolves(); storeAppStateEventStub = sinon.stub().resolves(); broadcastMessageToAllStub = sinon.stub().resolves('signed'); @@ -498,6 +502,38 @@ describe('peerNotification tests', () => { expect(message.apps.map((a) => a.name)).to.deep.equal(['app1']); }); + it('reads no placement record before every app installed at start has one', async () => { + let openBackfillGate; + waitForBackfillStub.callsFake(() => new Promise((resolve) => { openBackfillGate = resolve; })); + + const callPromise = peerNotification.checkAndNotifyPeersOfRunningApps(); + await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(placementsOfStub.called, 'read the records before the backfill finished').to.be.false; + expect(broadcastMessageToAllStub.called).to.be.false; + + openBackfillGate(); + await callPromise; + expect(placementsOfStub.calledOnce).to.be.true; + expect(broadcastMessageToAllStub.calledOnce).to.be.true; + }); + + it('announces as soon as this node learns its address', async () => { + let learnAddress; + waitForLocalSocketAddressStub.callsFake(() => new Promise((resolve) => { learnAddress = resolve; })); + + const callPromise = peerNotification.checkAndNotifyPeersOfRunningApps(); + await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(broadcastMessageToAllStub.called, 'broadcast before the node knew its address').to.be.false; + + learnAddress('192.168.1.1:16127'); + await callPromise; + expect(broadcastMessageToAllStub.calledOnce).to.be.true; + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.ip).to.equal('192.168.1.1:16127'); + }); + // g:/r: apps needed a carve-out while the set came from run-state, because a // masterSlave app stops its slave components deliberately. They need none now. it('announces g:/r: apps with stopped components, without a carve-out', async () => { diff --git a/tests/unit/placementsMigration.test.js b/tests/unit/placementsMigration.test.js new file mode 100644 index 0000000000..e75f504b45 --- /dev/null +++ b/tests/unit/placementsMigration.test.js @@ -0,0 +1,41 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('placements migration', () => { + let migration; + let learnAddress; + let findInDatabaseStub; + let backfillPlacementsStub; + + beforeEach(() => { + findInDatabaseStub = sinon.stub().resolves([{ name: 'web' }, { name: 'db' }]); + backfillPlacementsStub = sinon.stub().resolves({ created: 2, fromLocationRows: 2 }); + migration = proxyquire('../../ZelBack/src/services/migrations/placements', { + config: { database: { appslocal: { database: 'localzelapps' } } }, + '../dbHelper': { + databaseConnection: () => ({ db: () => ({}) }), + findInDatabase: findInDatabaseStub, + }, + '../fluxNetworkHelper': { + waitForLocalSocketAddress: () => new Promise((resolve) => { learnAddress = resolve; }), + }, + '../appManagement/appPlacements': { backfillPlacements: backfillPlacementsStub }, + '../utils/appConstants': { localAppsInformation: 'zelappsinformation' }, + }); + }); + + // The rows a record is dated from are filed under this node's address, so no + // record is written before the node knows it. + it('writes nothing until this node knows its address, then backfills the installed apps from it', async () => { + const running = migration.backfillPlacements(); + await new Promise((resolve) => { setImmediate(resolve); }); + sinon.assert.notCalled(backfillPlacementsStub); + + learnAddress('203.0.113.40:16127'); + await running; + + sinon.assert.calledOnceWithExactly(backfillPlacementsStub, ['web', 'db'], '203.0.113.40:16127'); + expect(findInDatabaseStub.firstCall.args[1]).to.equal('zelappsinformation'); + }); +}); From 0091b9f3a6ac7bc4a798997cdd2e31a00155f57f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:55:07 +0100 Subject: [PATCH 352/369] fix(network): a node asks whether it can be heard the moment it has an address after having none Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxNetworkHelper.js | 15 +++++++- .../src/services/nodeConfirmationService.js | 28 +++++++++++++++ tests/unit/fluxNetworkHelper.test.js | 13 +++++++ tests/unit/nodeConfirmationService.test.js | 36 +++++++++++++++++++ 4 files changed, 91 insertions(+), 1 deletion(-) diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 21427ae174..376dc15217 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -170,7 +170,8 @@ const EGRESS_TARGET = '203.0.113.1'; // localSocketAddress but gives the node no new address, so this keeps it. let lastKnownSocketAddress = null; // Emits 'change' with the new address when the node learns one that differs -// from lastKnownSocketAddress. +// from lastKnownSocketAddress, and 'known' with it when the node has an address +// after having none. const localSocketAddressEvents = new EventEmitter(); // Open while localSocketAddress holds an address. const localSocketAddressGate = new AsyncGate(); @@ -990,6 +991,7 @@ async function keepUPNPPortsOpen(req, res) { * @param {string} value ip or ip:port to be set (normalized to ip:port) */ function setLocalSocketAddress(value) { + const wasKnown = localSocketAddressGate.ready; localSocketAddress = value ? normalizeSocketAddress(value) : null; // (Re)set the freshness deadline whenever the value changes, so getLocalSocketAddress // serves it without a benchmark RPC until it goes stale (cleared when set to null). @@ -1005,6 +1007,7 @@ function setLocalSocketAddress(value) { lastKnownSocketAddress = localSocketAddress; localSocketAddressEvents.emit('change', localSocketAddress); } + if (localSocketAddress && !wasKnown) localSocketAddressEvents.emit('known', localSocketAddress); } /** @@ -1048,6 +1051,15 @@ function offLocalSocketAddressChange(listener) { localSocketAddressEvents.off('change', listener); } +/** + * @param {function(string): void} listener Called with the address each time the + * node has one after having none. + * @returns {void} + */ +function onLocalSocketAddressKnown(listener) { + localSocketAddressEvents.on('known', listener); +} + /** * Setter for dosMessage. * Main goal for this is testing availability. @@ -3752,6 +3764,7 @@ module.exports = { waitForLocalSocketAddress, onLocalSocketAddressChange, offLocalSocketAddressChange, + onLocalSocketAddressKnown, getFluxNodePrivateKey, getFluxNodePublicKey, MAX_KEEPALIVE_PORTS, diff --git a/ZelBack/src/services/nodeConfirmationService.js b/ZelBack/src/services/nodeConfirmationService.js index af58554dbc..f0de67e3ef 100644 --- a/ZelBack/src/services/nodeConfirmationService.js +++ b/ZelBack/src/services/nodeConfirmationService.js @@ -15,6 +15,7 @@ let daemonStale = false; let messageCapable = false; let started = false; let lastSuccessfulPoll = null; +let capabilityCheck = Promise.resolve(); const confirmedGate = new AsyncGate(); const confirmationStatusGate = new AsyncGate(); @@ -123,6 +124,23 @@ async function poll() { return; } + await checkMessageCapability(); +} + +/** + * Whether this node can be heard: the address benchmark reports is listed under + * this node's key. Told to the capability listeners when it changes. One check + * runs at a time, so a check started by a poll and one started by the node + * learning its address cannot both report the same change. + * @returns {Promise} + */ +function checkMessageCapability() { + capabilityCheck = capabilityCheck.then(readMessageCapability, readMessageCapability); + return capabilityCheck; +} + +async function readMessageCapability() { + const prevMessageCapable = messageCapable; let newMessageCapable = false; try { if (!ourPubkey) { @@ -161,6 +179,15 @@ async function poll() { } } +/** + * The node has an address after having none: whether it can be heard is asked + * at once. + * @returns {void} + */ +function onOwnAddressKnown() { + if (daemonConfirmed) checkMessageCapability().catch((error) => log.error(error)); +} + function scheduleNext() { setTimeout(async () => { await poll(); @@ -171,6 +198,7 @@ function scheduleNext() { async function start() { if (started) return; started = true; + fluxNetworkHelper.onLocalSocketAddressKnown(onOwnAddressKnown); await poll(); confirmationStatusGate.open(); scheduleNext(); diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index abdde5a462..8cbcf2f4eb 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -436,6 +436,19 @@ describe('fluxNetworkHelper tests', () => { expect(answer).to.equal('203.0.113.41:16127'); }); + it('tells its listeners each time the node has an address after having none', () => { + const heard = []; + const listener = (address) => heard.push(address); + fluxNetworkHelper.onLocalSocketAddressKnown(listener); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.44:16127'); + fluxNetworkHelper.setLocalSocketAddress('203.0.113.44:16127'); + fluxNetworkHelper.setLocalSocketAddress(null); + fluxNetworkHelper.setLocalSocketAddress('203.0.113.44:16127'); + + expect(heard).to.deep.equal(['203.0.113.44:16127', '203.0.113.44:16127']); + }); + it('waits again when the address it was woken for is gone before it reads it', async () => { benchStub.resolves({ status: 'error' }); let answer = null; diff --git a/tests/unit/nodeConfirmationService.test.js b/tests/unit/nodeConfirmationService.test.js index 8f3e8cec65..8f782c3e85 100644 --- a/tests/unit/nodeConfirmationService.test.js +++ b/tests/unit/nodeConfirmationService.test.js @@ -15,6 +15,7 @@ describe('nodeConfirmationService', () => { let getFluxNodePublicKeyStub; let getFluxnodeBySocketAddressStub; let logStub; + let ownAddressKnown; beforeEach(() => { clock = sinon.useFakeTimers({ shouldAdvanceTime: false }); @@ -29,6 +30,7 @@ describe('nodeConfirmationService', () => { './fluxNetworkHelper': { getLocalSocketAddress: getLocalSocketAddressStub, getFluxNodePublicKey: getFluxNodePublicKeyStub, + onLocalSocketAddressKnown: (listener) => { ownAddressKnown = listener; }, }, './networkStateService': { getFluxnodeBySocketAddress: getFluxnodeBySocketAddressStub }, '../lib/log': logStub, @@ -155,6 +157,40 @@ describe('nodeConfirmationService', () => { }); describe('onMessageCapabilityChange', () => { + it('is heard the moment the node learns its address', async () => { + const callback = sinon.spy(); + service.onMessageCapabilityChange(callback); + setupConfirmed(); + getLocalSocketAddressStub.resolves(null); + await service.start(); + expect(service.canSendMessages()).to.be.false; + + getLocalSocketAddressStub.resolves('1.2.3.4:16127'); + ownAddressKnown('1.2.3.4:16127'); + await clock.tickAsync(0); + + expect(service.canSendMessages()).to.be.true; + sinon.assert.calledOnceWithExactly(callback, true); + }); + + it('reports one change when a poll checks while the address arriving is still being checked', async () => { + const callback = sinon.spy(); + service.onMessageCapabilityChange(callback); + setupConfirmed(); + getLocalSocketAddressStub.resolves(null); + await service.start(); + + let answerFirstCheck; + getLocalSocketAddressStub.onCall(1).returns(new Promise((resolve) => { answerFirstCheck = resolve; })); + getLocalSocketAddressStub.resolves('1.2.3.4:16127'); + ownAddressKnown('1.2.3.4:16127'); + await advancePoll(); + answerFirstCheck('1.2.3.4:16127'); + await clock.tickAsync(0); + + sinon.assert.calledOnceWithExactly(callback, true); + }); + it('should fire callback on false→true transition', async () => { const callback = sinon.spy(); service.onMessageCapabilityChange(callback); From d65cc5cba3891eb293caec5532ee844960d97f5b Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 12:56:51 +0100 Subject: [PATCH 353/369] test(harness): 3506 a node that boots without its address dates its records from its own row once it has one Co-Authored-By: Claude Opus 5.5 --- ...dress-dates-its-records-once-it-has-one.js | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js diff --git a/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js b/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js new file mode 100644 index 0000000000..2dcaedaff1 --- /dev/null +++ b/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js @@ -0,0 +1,121 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { waitForBootSettled, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { restartFluxos } from '../framework/container.js'; +import { setBenchmarkUnresolved } from '../framework/daemon-control.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node's location rows are filed under its address, and an installed app +// without a placement record is dated from this node's own row. A node that +// boots before benchmark reports its address writes no record until it has one +// (app:placementRecorded), then dates the app at its own row, and its next +// announcement carries that date to its peers. + +const REAL_NODES = 3; +const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); +const DAY_MS = 24 * 60 * 60 * 1000; +// Production's location lifetime, so this node's own row still stands while it +// has no address. +const LOCATION_TTL_S = 30 * 60; +// Long enough that the node is not out of service while the case holds its +// address back, at the harness's 15 s ask interval. +const PUBLIC_IP_ASKS_BEFORE_DOS = 40; +// One harness ask interval and three confirmation polls: a backfill that ran +// without the address has written by then. +const UNRESOLVED_WINDOW_MS = 20000; +const bare = (address) => String(address).replace(/:16127$/, ''); + +describe('a node that boots without its address dates its records once it has one', function () { + let env; + dumpLogsOnFailure(() => env); + let idx; + let holder; + let others; + const appName = `e2enoaddress${Date.now()}`; + + const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); + const runningSinceIn = (data) => { + const entry = data.apps?.find((a) => a.name === appName); + return entry ? new Date(entry.runningSince).getTime() : null; + }; + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ + hookCtx: this, + nodes: REAL_NODES, + tickerAutostart: false, + configOverrides: { fluxapps: { locationTtlS: LOCATION_TTL_S, publicIpAsksBeforeDos: PUBLIC_IP_ASKS_BEFORE_DOS } }, + }); + await bootAndPeer(env); + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ + name: appName, + instances: 1, + compose: [{ + name: 'alpha', + description: 'no address at boot', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + idx = await seedAndInstall(env, app); + holder = socketAddr(idx + 1); + others = realNodes.filter((i) => i !== idx); + await env.clients[idx].waitForEvent('app:running', (d) => runningSinceIn(d) !== null, 120000); + }); + + after(async function () { + this.timeout(60000); + if (holder) await setBenchmarkUnresolved(holder, false).catch(() => {}); + await env?.teardown(); + }); + + it('writes no record while it has no address, then dates the app at its own row and announces that', async function () { + this.timeout(480000); + const rowSince = Date.now() - 3 * DAY_MS; + await dbClient(idx + 1).deletePlacement(appName); + await dbClient(idx + 1).setAppLocationRunningSince(appName, holder, rowSince); + expect((await dbClient(idx + 1).appLocationAt(appName, holder))?.runningSince?.getTime(), 'fixture: the node\'s own row').to.equal(rowSince); + + await setBenchmarkUnresolved(holder, true); + const afterIds = marks(); + await restartFluxos(env.clients[idx].container); + await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); + + await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, UNRESOLVED_WINDOW_MS, { afterId: afterIds[idx] }); + + const resolvedIds = marks(); + await setBenchmarkUnresolved(holder, false); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 120000, { afterId: resolvedIds[idx] }); + expect(recorded.source, 'the record was not dated from the node\'s own row').to.equal('locationRow'); + expect(new Date(recorded.placedAt).getTime(), 'the record is not the node\'s own row\'s date').to.equal(rowSince); + + const stored = await Promise.all(others.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), + 120000, + { afterId: resolvedIds[i] }, + ))); + expect(stored.map(({ data }) => runningSinceIn(data)), 'a peer stored a different runningSince').to.deep.equal(others.map(() => rowSince)); + }); +}); From e8acbac04714505f7c6d9fb0ea4ad805486a231f Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:08:44 +0100 Subject: [PATCH 354/369] test(harness): 3506 holds the address back through boot, with a boot sync limit that keeps the app Co-Authored-By: Claude Opus 5.5 --- ...t-its-address-dates-its-records-once-it-has-one.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js b/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js index 2dcaedaff1..5c1b038ee6 100644 --- a/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js +++ b/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js @@ -30,6 +30,10 @@ const LOCATION_TTL_S = 30 * 60; // Long enough that the node is not out of service while the case holds its // address back, at the harness's 15 s ask interval. const PUBLIC_IP_ASKS_BEFORE_DOS = 40; +// A node syncs its apps only once it can be heard, which needs its address, and +// removes them when it has not synced within this: long enough to hold the +// address back and keep the app. +const BOOT_SYNC_TIMEOUT_MS = 300000; // One harness ask interval and three confirmation polls: a backfill that ran // without the address has written by then. const UNRESOLVED_WINDOW_MS = 20000; @@ -55,7 +59,10 @@ describe('a node that boots without its address dates its records once it has on hookCtx: this, nodes: REAL_NODES, tickerAutostart: false, - configOverrides: { fluxapps: { locationTtlS: LOCATION_TTL_S, publicIpAsksBeforeDos: PUBLIC_IP_ASKS_BEFORE_DOS } }, + configOverrides: { + fluxapps: { locationTtlS: LOCATION_TTL_S, publicIpAsksBeforeDos: PUBLIC_IP_ASKS_BEFORE_DOS }, + system: { bootSyncTimeoutMs: BOOT_SYNC_TIMEOUT_MS }, + }, }); await bootAndPeer(env); await pushImage(appName, 'v1'); @@ -100,12 +107,12 @@ describe('a node that boots without its address dates its records once it has on await setBenchmarkUnresolved(holder, true); const afterIds = marks(); await restartFluxos(env.clients[idx].container); - await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, UNRESOLVED_WINDOW_MS, { afterId: afterIds[idx] }); const resolvedIds = marks(); await setBenchmarkUnresolved(holder, false); + await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 120000, { afterId: resolvedIds[idx] }); expect(recorded.source, 'the record was not dated from the node\'s own row').to.equal('locationRow'); expect(new Date(recorded.placedAt).getTime(), 'the record is not the node\'s own row\'s date').to.equal(rowSince); From 02a0e52e31da8956f755580eeaf53d5ce66df6c0 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:25:50 +0100 Subject: [PATCH 355/369] fix(apps): a removal stands when its placement record cannot be deleted Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/appLifecycle/appUninstaller.js | 10 ++++++++-- ZelBack/src/services/appManagement/appPlacements.js | 6 +++--- tests/unit/appUninstaller.test.js | 10 ++++++++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 5efb6f8633..84fbc846d1 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -1149,8 +1149,14 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - // the app's volume is gone, and the record of when this node took it with it - await appPlacements.removePlacement(appName); + // The app's volume is gone, and the record of when this node took it with it. + // A record left behind is never read, as the app is not installed, and boot + // removes it, so the removal stands whether this delete succeeds or not. + try { + await appPlacements.removePlacement(appName); + } catch (error) { + log.warn(`removeAppLocally - the placement record of ${appName} was not removed: ${error.message}`); + } // The app is gone for good: nothing reconciles an app with no row, so its // removal records have no reader left. Only this full-uninstall path clears // them - softRemoveAppLocally deletes the row too, but as one step of a diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js index 60130311e3..774ee7afb9 100644 --- a/ZelBack/src/services/appManagement/appPlacements.js +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -16,9 +16,9 @@ const { AsyncGate } = require('../utils/asyncGate'); // expiry, eviction, sigterm - while the app goes on running. // // A record outlives its app when a soft redeploy removes the app and its reinstall -// does not happen, which leaves the volume on disk and the record with it, or when -// the app is removed while the backfill below is writing. Boot removes records of -// apps not installed. Only an installed app's record is read, and the app returns +// does not happen, which leaves the volume on disk and the record with it, when +// the app is removed while the backfill below is writing, or when the removal +// cannot delete it. Boot removes records of apps not installed. Only an installed app's record is read, and the app returns // only through a hard install, which overwrites it. // // Kept apart from the app's specification in appsInformation: a spec update diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index 8b6c868db8..ef6637e206 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -1041,6 +1041,16 @@ describe('appUninstaller tests', () => { sinon.assert.calledOnceWithExactly(appPlacementsStub.removePlacement, 'testapp'); }); + it('a record that cannot be deleted leaves the removal standing', async () => { + const uninstaller = buildUninstaller(composedSpec); + appPlacementsStub.removePlacement.rejects(new Error('mongo went away')); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, true); + + expect(outcome).to.equal(RemovalOutcome.REMOVED); + expect(logStub.warn.calledWithMatch(/placement record of testapp was not removed: mongo went away/)).to.equal(true); + }); + it('stays when one component is removed and the app with it is not', async () => { const uninstaller = buildUninstaller(composedSpec); From 29bdabf4ea384756c683fd820502aef5df851ab3 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:29:18 +0100 Subject: [PATCH 356/369] fix(network): network:apprunning is published only for a message that wrote, refused, cleared or relayed something Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/fluxCommunication.js | 18 ++++++---- tests/unit/fluxCommunication.test.js | 40 ++++++++++++++++++++++- 2 files changed, 50 insertions(+), 8 deletions(-) diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 066b4865d0..d31aa5a960 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -478,13 +478,17 @@ async function handleAppRunningMessage(message, fromIP, port) { } // Everything this node did with the message: the entries it wrote, the names // it refused, whether it cleared the sender's rows, and whether it passed it on. - fluxEventBus.publish('network:apprunning', { - ip: message.data.ip, - apps: result.written, - refused: result.refused, - cleared: result.cleared, - relayed, - }); + // A message that did none of these, such as a copy of one already stored, + // publishes nothing. + if (result.written.length || result.refused.length || result.cleared || relayed) { + fluxEventBus.publish('network:apprunning', { + ip: message.data.ip, + apps: result.written, + refused: result.refused, + cleared: result.cleared, + relayed, + }); + } } catch (error) { log.error(error); } diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index f03127bb21..aa0b2aa856 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -28,6 +28,7 @@ const { peerManager } = require('../../ZelBack/src/services/utils/peerState'); const { PEER_SOURCE } = require('../../ZelBack/src/services/utils/FluxPeerSocket'); const rateLimit = require('../../ZelBack/src/services/utils/rateLimit'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); +const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); let localWsServer; let localWsUrl; @@ -552,7 +553,9 @@ describe('fluxCommunication tests', () => { }); it('should broadcast the app message if a proper data is given', async () => { - sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ stored: true, rebroadcast: true }); + sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ + stored: true, rebroadcast: true, written: [{ name: 'myApp', hash: 'myapphash' }], refused: [], cleared: false, + }); sinon.stub(messageStore, 'storeAppStateEvent'); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 0 } }); const fromIp = '127.0.0.5'; @@ -630,6 +633,41 @@ describe('fluxCommunication tests', () => { sinon.assert.notCalled(broadcastHashSpy); }).timeout(5000); + const runningMessage = () => ({ + data: { + type: 'fluxapprunning', version: 2, apps: [{ name: 'myApp', hash: 'myapphash' }], broadcastedAt: Date.now(), ip: '203.0.113.31:16127', + }, + timestamp: Date.now(), + }); + const apprunningPublished = (publish) => publish.getCalls().filter((call) => call.args[0] === 'network:apprunning'); + + it('publishes nothing for a message that wrote, refused, cleared and relayed nothing', async () => { + sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ + stored: false, rebroadcast: false, written: [], refused: [], cleared: false, + }); + sinon.stub(messageStore, 'storeAppStateEvent'); + const publish = sinon.stub(fluxEventBus, 'publish'); + + await fluxCommunication.handleAppRunningMessage(runningMessage(), '127.0.0.5', '16127'); + + expect(apprunningPublished(publish)).to.have.lengthOf(0); + }); + + it('publishes the names it refused, though it wrote and relayed nothing', async () => { + sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ + stored: false, rebroadcast: false, written: [], refused: ['unhashed'], cleared: false, + }); + sinon.stub(messageStore, 'storeAppStateEvent'); + const publish = sinon.stub(fluxEventBus, 'publish'); + + await fluxCommunication.handleAppRunningMessage(runningMessage(), '127.0.0.5', '16127'); + + const [call] = apprunningPublished(publish); + expect(call.args[1]).to.deep.equal({ + ip: '203.0.113.31:16127', apps: [], refused: ['unhashed'], cleared: false, relayed: false, + }); + }); + // A refused message is logged where it arrived, and goes no further. it('logs a message its store refuses, and does not relay it', async () => { sinon.stub(messageStore, 'storeAppRunningMessage').resolves(new Error('Invalid Flux App Running message for storing')); From a815997c26f22c14f7a25481b79157a440b812f7 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 13:36:15 +0100 Subject: [PATCH 357/369] fix(apps): an app not yet paid for is tested by the owner whose key signs its message Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appInstaller.js | 21 ++++++- tests/unit/appInstaller.test.js | 62 +++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 4163f6741e..4ada81c87d 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -1176,6 +1176,20 @@ async function checkAppRequirements(appSpecs, skipGeolocation = false, skipStati return true; } +/** + * The owner whose key signs a pending app message: the owner a registration + * names, and for an update the owner of the app as registered, who signs the + * update whatever owner it names. + * @param {object} tempMessage A temporary app message. + * @returns {Promise} The signer's address, or null when it cannot be found. + */ +async function pendingMessageSigner(tempMessage) { + const isUpdate = tempMessage.type === 'fluxappupdate' || tempMessage.type === 'zelappupdate'; + if (!isUpdate) return tempMessage.appSpecifications?.owner ?? null; + const registered = await getApplicationGlobalSpecifications(tempMessage.appSpecifications?.name); + return registered?.owner ?? null; +} + /** * Test application installation - Similar to installAppLocally but for testing with reduced resource requirements * @param {object} req - Request object containing appname in params or query @@ -1205,12 +1219,13 @@ async function testAppInstall(req, res) { } let appSpecifications; - // An app not yet paid for is tested by its owner, addressed by its message - // hash. The flux team may test any. + // An app not yet paid for is tested by the owner whose key signs its + // message, addressed by its message hash. The flux team may test any. const tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(appname); if (tempMessage) { const caller = serviceHelper.ensureObject(authOf(req))?.zelid; - const mayTest = caller === tempMessage.appSpecifications?.owner + const signer = await pendingMessageSigner(tempMessage); + const mayTest = (Boolean(caller) && caller === signer) || await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (!mayTest) { const errMessage = messageHelper.errUnauthorizedMessage(); diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 53cab33cee..87e54500f9 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -591,6 +591,7 @@ describe('appInstaller tests', () => { const owner = '1OwnerOfTheTemporaryApp'; const tempMessage = { hash: 'temphash', + type: 'fluxappregister', appSpecifications: { name: 'trialapp', version: 4, owner, compose: [{ name: 'web', repotag: 'org/web:1', cpu: 0.5, ram: 500, hdd: 5 }], }, @@ -657,6 +658,67 @@ describe('appInstaller tests', () => { expect(res.json.firstCall.args[0].data.message).to.equal('Flux App trialapp is installed on this node. Test it on a node that does not hold it.'); }); + // An update is signed by the app's owner as registered, whatever owner it names. + describe('an update that hands the app to a new owner', () => { + const current = '1CurrentOwner'; + const named = '1NewOwnerNamedInTheUpdate'; + + beforeEach(() => { + messageVerifierStub.checkAppTemporaryMessageExistence.withArgs('temphash').resolves({ + ...tempMessage, type: 'fluxappupdate', appSpecifications: { ...tempMessage.appSpecifications, owner: named }, + }); + baseStubs['../appDatabase/registryManager'].getApplicationGlobalSpecifications + .withArgs('trialapp').resolves({ ...tempMessage.appSpecifications, owner: current }); + }); + + it('lets the current owner test it', async () => { + const req = requestAs(current); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(true); + }); + + it('refuses the owner it names, who does not own the app yet', async () => { + const req = requestAs(named); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); + }); + + it('refuses a session that names no address when the app is not registered', async () => { + baseStubs['../appDatabase/registryManager'].getApplicationGlobalSpecifications.withArgs('trialapp').resolves(null); + const req = requestAs(null); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + }); + + it('refuses everyone but the flux team when the app is not registered', async () => { + baseStubs['../appDatabase/registryManager'].getApplicationGlobalSpecifications.withArgs('trialapp').resolves(null); + const req = requestAs(current); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + }); + }); + it('refuses a caller with no session at all', async () => { const req = { params: { appname: 'temphash' }, query: {} }; const res = response(); From da0784110573fbe0b762c09f36b8391a39099122 Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 15:44:09 +0100 Subject: [PATCH 358/369] fix(apps): an announcement holds up removals only from its snapshot to its broadcast Co-Authored-By: Claude Opus 5.5 --- .../services/appMessaging/peerNotification.js | 30 +++++++++++++------ tests/unit/peerNotification.test.js | 17 +++++++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index 9c7f43f0e2..7737b4b44b 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -35,11 +35,14 @@ let overrunning = false; // loop survives its own abort. Nothing resets this. let broadcasting = false; -// Held for the whole of a cycle, so a stop can wait for the cycle in flight -// rather than returning while it is still running. A teardown that returns -// before the thing is torn down is the same lie as a stop that does not stop. -// A removal waits on it too, which is why it lives in globalState. +// Held from a cycle's snapshot to its broadcast. A removal waits on it, so its +// message goes out after an announcement that took its list before the removal, +// which is why it lives in globalState. const cycleLock = globalState.announceCycle; +// Settles when the cycle in flight, if any, has finished, so a stop can wait for +// it rather than returning while it is still running. A teardown that returns +// before the thing is torn down is the same lie as a stop that does not stop. +let cycleFinished = Promise.resolve(); /** * Schedule the next announcement so that the PERIOD is fixed, rather than the @@ -115,7 +118,7 @@ async function stopBroadcasting() { // The queued repeat goes with the stop. Left set, it is a cycle the next // start would run before the one it schedules for itself. rebroadcastNeeded = false; - await cycleLock.waitReady(); + await cycleFinished; } function initialize() { @@ -134,13 +137,15 @@ async function checkAndNotifyPeersOfRunningApps() { return; } broadcastInProgress = true; + let finishCycle; + cycleFinished = new Promise((resolve) => { finishCycle = resolve; }); // Taken before any work, so the schedule below subtracts the WHOLE cycle - // including the paths that give up early, which cost time too. const startedAt = process.hrtime.bigint(); - await cycleLock.enable(); + let locked = false; try { - // Observation only. The lock above is what a caller waiting on a cycle in - // flight actually waits for. + // Observation only. The lock taken below is what a caller waiting on a cycle + // in flight actually waits for. fluxEventBus.publish('app:announcing', {}); if (!nodeConfirmationService.canSendMessages()) { log.info('checkAndNotifyPeersOfRunningApps - Node cannot send messages, skipping broadcast'); @@ -158,6 +163,12 @@ async function checkAndNotifyPeersOfRunningApps() { const localSocketAddr = await fluxNetworkHelper.waitForLocalSocketAddress(); + // A cycle is in flight from the snapshot to the broadcast, and a removal waits + // for it so the two go out in the order they happened. The waits above are + // outside it, so a node waiting for its address holds up no removal. + await cycleLock.enable(); + locked = true; + // Stamped where the snapshot is taken, not where the message is built: every // consumer reads broadcastedAt as the time these apps were installed here, and // a stamp taken after the per-app reads below would out-rank a removal that @@ -287,7 +298,8 @@ async function checkAndNotifyPeersOfRunningApps() { log.error(error); } finally { broadcastInProgress = false; - cycleLock.disable(); + if (locked) cycleLock.disable(); + finishCycle(); if (rebroadcastNeeded) { rebroadcastNeeded = false; setImmediate(() => checkAndNotifyPeersOfRunningApps()); diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index 6fdf2f3cd4..37e46bd94b 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -534,6 +534,23 @@ describe('peerNotification tests', () => { expect(message.ip).to.equal('192.168.1.1:16127'); }); + // A removal waits for a cycle in flight; a cycle still waiting for the node's + // address has taken no snapshot and holds up no removal. + it('is not in flight while it waits for the node\'s address', async () => { + let learnAddress; + waitForLocalSocketAddressStub.callsFake(() => new Promise((resolve) => { learnAddress = resolve; })); + + const callPromise = peerNotification.checkAndNotifyPeersOfRunningApps(); + await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(announceCycle.locked, 'in flight while waiting for the address').to.equal(false); + + learnAddress('192.168.1.1:16127'); + await callPromise; + expect(announceCycle.locked).to.equal(false); + expect(broadcastMessageToAllStub.calledOnce).to.be.true; + }); + // g:/r: apps needed a carve-out while the set came from run-state, because a // masterSlave app stops its slave components deliberately. They need none now. it('announces g:/r: apps with stopped components, without a carve-out', async () => { From 465206ff425fd30553539aefb21a458dc70ef67c Mon Sep 17 00:00:00 2001 From: David White Date: Thu, 22 Oct 2026 15:51:40 +0100 Subject: [PATCH 359/369] fix(apps): a removal tells the network after the announcement in flight, however long it takes Co-Authored-By: Claude Opus 5.5 --- .../src/services/appLifecycle/appUninstaller.js | 9 +++++---- ZelBack/src/services/utils/appConstants.js | 9 --------- tests/unit/appUninstaller.test.js | 17 ----------------- 3 files changed, 5 insertions(+), 30 deletions(-) diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 84fbc846d1..785174bb9f 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -10,7 +10,7 @@ const globalState = require('../utils/globalState'); const folderStates = require('../appMonitoring/folderStates'); const log = require('../../lib/log'); const { - localAppsInformation, globalAppsInformation, globalAppsMessages, ANNOUNCE_CYCLE_WAIT_MS, + localAppsInformation, globalAppsInformation, globalAppsMessages, } = require('../utils/appConstants'); const config = require('config'); // const advancedWorkflows = require('./advancedWorkflows'); // Moved to dynamic require to avoid circular dependency @@ -110,9 +110,10 @@ async function announceRemoval(appName) { // app still names it, and a claim arriving after this message re-creates the // row it cleared. Waiting for the cycle in flight puts the two on the wire in // the order they happened, so the claim is applied first and this clears it. - // A cycle starting from here on reads the mark and leaves the app out. - // Bounded: a wedged cycle must not hold up the node's removals. - await globalState.announceCycle.readyTimeout(ANNOUNCE_CYCLE_WAIT_MS); + // A cycle starting from here on reads the mark and leaves the app out. A cycle + // is in flight only from its snapshot to its broadcast, and every call in that + // span has its own time limit. + await globalState.announceCycle.waitReady({ waitAll: false }); const ip = await fluxNetworkHelper.getLocalSocketAddress(); if (!ip) { log.warn(`${appName} removed without announcing it - this node's own address is unknown, so peers hold its location row until it expires`); diff --git a/ZelBack/src/services/utils/appConstants.js b/ZelBack/src/services/utils/appConstants.js index e19b21934e..6facb02a2f 100644 --- a/ZelBack/src/services/utils/appConstants.js +++ b/ZelBack/src/services/utils/appConstants.js @@ -139,14 +139,6 @@ const ANNOUNCE_INTERVAL_MS = Math.floor( (RUNNING_EXPIRY_MS * (1 - ANNOUNCE_SLACK)) / ANNOUNCES_PER_EXPIRY / 1000, ) * 1000; -// How long a removal waits for an announcement cycle to finish sending before it -// tells the network the app is gone. A cycle is one database read per installed -// app, one signature and one broadcast, so this is orders of magnitude above a -// healthy one at any plausible app count - and it is bounded at all because a -// wedged cycle must not hold up the node's removals, which take the removal lock -// and block every install and redeploy behind them. -const ANNOUNCE_CYCLE_WAIT_MS = 30 * 1000; - // Hash sync constants (blocks, at 30s per block) const HASH_EXPIRY_BLOCKS = 1051200; // ~1 year — permanently flag unresolvable hashes const HASH_RETRY_BACKOFF = [0, 100, 500, 2500, 12500, 50000, 100000]; // ~0, 50min, 4h, 21h, 4d, 17d, 35d @@ -195,7 +187,6 @@ module.exports = { GOSSIP_VALIDITY_MS, RUNNING_EXPIRY_MS, ANNOUNCE_INTERVAL_MS, - ANNOUNCE_CYCLE_WAIT_MS, INSTALLING_EXPIRY_MS, INSTALLING_ERRORS_EXPIRY_MS, SIGTERM_EXPIRY_MS, diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index ef6637e206..dbab26ad0e 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -1267,23 +1267,6 @@ describe('appUninstaller tests', () => { expect(getLocalSocketAddressStub.called, 'the removal never reached its broadcast').to.be.true; }); - // A cycle that never finishes must not hold the node's removals: they take the - // removal lock, and every install and redeploy queues behind that. - it('gives up on a wedged cycle and announces anyway', async () => { - const uninstaller = buildUninstaller(v2Spec, { ANNOUNCE_CYCLE_WAIT_MS: 50 }); - await announceCycle.enable(); - - const removal = uninstaller.removeAppLocally('testapp', res, true, true, true); - const raced = await Promise.race([ - removal.then(() => 'announced'), - new Promise((resolve) => { setTimeout(() => resolve('stuck'), 2000); }), - ]); - - expect(raced, 'a wedged announcement cycle held the removal indefinitely').to.equal('announced'); - expect(getLocalSocketAddressStub.called, 'gave up on the wait without going on to broadcast').to.be.true; - announceCycle.disable(); - }); - // A redeploy tells the network nothing, so it contradicts no announcement and // must not queue behind one - every spec change on the node would pay for it. it('does not wait for a cycle when the removal says nothing to the network', async () => { From 29087999daedec4dc06412b1d903b4e9cee29d60 Mon Sep 17 00:00:00 2001 From: David White Date: Fri, 23 Oct 2026 14:54:46 +0100 Subject: [PATCH 360/369] fix(apps): the expiry check leaves an app being tested installed Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 3 +++ tests/unit/registryManager.test.js | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index d608a7a912..a9cff1c4b0 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -1758,6 +1758,9 @@ async function expireGlobalApplications() { const appsToRemove = []; const locallyExpired = []; appsInstalled.forEach((app) => { + // An app being tested has no height until it is registered, and its test + // removes it when it ends. + if (globalState.testInstallingApps.has(app.name)) return; if (appNamesToExpire.includes(app.name)) { appsToRemove.push(app); } else if (!app.height) { diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 0868e6e8ad..7ba91e4659 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1092,6 +1092,29 @@ describe('registryManager tests', () => { expect(queued()).to.deep.equal(['DeadApp']); }); + describe('an app with no height', () => { + afterEach(() => { + globalState.testInstallingApps.clear(); + }); + + it('should leave an app being tested installed', async () => { + installed = [spec('TestedApp', 't', undefined, 100)]; + globalState.testInstallingApps.add('TestedApp'); + + await registryManager.expireGlobalApplications(); + + expect(queued()).to.deep.equal([]); + }); + + it('should uninstall one no test is running for', async () => { + installed = [spec('LeftApp', 't', undefined, 100)]; + + await registryManager.expireGlobalApplications(); + + expect(queued()).to.deep.equal(['LeftApp']); + }); + }); + it('should expire an app at its expiration height, installed or only global', async () => { // term ends at the tip await database.collection(appsInformation).insertMany([spec('EndsNowApp', 'n', 3007900, 100), spec('LiveApp', 'l', 3007900, 101)]); From e5f5caefdc5e619969d4c044ef20a30a86c168cc Mon Sep 17 00:00:00 2001 From: David White Date: Sat, 24 Oct 2026 06:34:51 +0100 Subject: [PATCH 361/369] test(hostStall): a held write is released once A held write's release started the real write each time it was called, and the block's afterEach releases every write again after a test has released its own. The extra writes were not awaited, so one could land while the outer afterEach removed the temporary directory, failing the rmdir with ENOTEMPTY. Co-Authored-By: Claude Opus 5.5 --- tests/unit/hostStallService.test.js | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js index 2febd02467..c3018b3525 100644 --- a/tests/unit/hostStallService.test.js +++ b/tests/unit/hostStallService.test.js @@ -439,7 +439,14 @@ describe('hostStallService tests', () => { holding = true; const writeFile = fs.promises.writeFile.bind(fs.promises); sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (holding && String(file).startsWith(paths.stateFile) - ? new Promise((resolve) => { releases.push(() => resolve(writeFile(file, ...rest))); }) + ? new Promise((resolve) => { + let released = false; + releases.push(() => { + if (released) return; + released = true; + resolve(writeFile(file, ...rest)); + }); + }) : writeFile(file, ...rest))); }); From 9dfef7f4f7989c6d2fc82d8766ff09e090453de9 Mon Sep 17 00:00:00 2001 From: Valter Silva Date: Sat, 24 Oct 2026 09:12:03 +0100 Subject: [PATCH 362/369] chore(release): bump version to 8.22.0 and carry the release fingerprint Steps 1 and 2 of RELEASING.md for the development -> master release PR. package.json moves 8.21.1 -> 8.22.0, which is what release-gate's "version moves forward" check reads. helpers/hashes.json gains 0840c013921a322079de7d694417ff42, the md5 over ./ZelBack at this tip. Neither file is inside ZelBack, so committing them does not move the fingerprint. Co-Authored-By: Claude Opus 5.5 --- helpers/hashes.json | 3 ++- package.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/helpers/hashes.json b/helpers/hashes.json index 8b80b18bfc..a53d06e051 100644 --- a/helpers/hashes.json +++ b/helpers/hashes.json @@ -13,5 +13,6 @@ "5bc27303e232f8bcbf1f531987e28f0a", "16379f485d76083906f1fd19790dd1c4", "adcaa6996078beaff5d62fe95b2774d2", - "b76e4ee20664bc6db7a91276f98f66b7" + "b76e4ee20664bc6db7a91276f98f66b7", + "0840c013921a322079de7d694417ff42" ] diff --git a/package.json b/package.json index dee54d682d..5ca8589a73 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flux", - "version": "8.21.1", + "version": "8.22.0", "description": "Flux, Your Gateway to a Decentralized World", "repository": { "type": "git", From 48a0add939dcd4fb9b04e17a14e88351f6108b83 Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 06:35:27 +0000 Subject: [PATCH 363/369] fix(apps): validate app name and owner Requests now refuse an app name that does not match the app-name rule and an owner that is not a Flux ID or an Ethereum address, using the existing APP_NAME_REGEX, MAX_APP_NAME_LENGTH and signatureVerifier.isValidSigningIdentity. Co-Authored-By: Claude Opus 5.5 --- ZelBack/src/services/benchmarkService.js | 62 +++++++++++++----- tests/unit/benchmarkService.test.js | 81 ++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 16 deletions(-) diff --git a/ZelBack/src/services/benchmarkService.js b/ZelBack/src/services/benchmarkService.js index e9303d5e23..67e15c07b0 100644 --- a/ZelBack/src/services/benchmarkService.js +++ b/ZelBack/src/services/benchmarkService.js @@ -11,7 +11,9 @@ const generalService = require('./generalService'); const upnpService = require('./upnpService'); const fluxRpc = require('./utils/fluxRpc'); const dbHelper = require('./dbHelper'); +const signatureVerifier = require('./signatureVerifier'); const { Privilege, authOf } = require('./utils/privileges'); +const { APP_NAME_REGEX, MAX_APP_NAME_LENGTH } = require('./utils/appConstants'); // eslint-disable-next-line no-unused-vars const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -279,44 +281,72 @@ async function signFluxTransactionPost(req, res) { }); } +/** + * Whether a request names a valid app and a valid owner: an app name + * as a spec allows it, and an owner that can sign (a Flux ID or an Ethereum address). + * @param {string} message JSON object with `appName` and `fluxID`. + * @returns {boolean} + */ +function isValidAppRequest(message) { + let request; + try { + request = JSON.parse(message); + } catch { + return false; + } + if (!request || typeof request !== 'object') return false; + + const { appName, fluxID } = request; + return typeof appName === 'string' + && appName.length <= MAX_APP_NAME_LENGTH + && APP_NAME_REGEX.test(appName) + && signatureVerifier.isValidSigningIdentity(fluxID); +} + +/** + * Ask FluxBench to handle an app request. A request without a valid app + * name and owner is refused here and never sent. + * @param {string} rpc FluxBench RPC. + * @param {string} message JSON object with `appName` and `fluxID`. + * @returns {Promise} Message. + */ +async function executeAppCall(rpc, message) { + if (!isValidAppRequest(message)) { + return messageHelper.createErrorMessage('Invalid app name or owner'); + } + return executeCall(rpc, [message]); +} + /** * Ask FLuxBench to decrypt message - * @param {object} message message object with information to be decrypted. + * @param {string} message JSON object with information to be decrypted. */ async function decryptMessage(message) { - const rpccall = 'decryptmessage'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('decryptmessage', message); } /** * Ask FLuxBench to decrypt rsa message - * @param {object} message message object with information to be decrypted. + * @param {string} message JSON object with information to be decrypted. */ async function decryptRSAMessage(message) { - const rpccall = 'decryptrsamessage'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('decryptrsamessage', message); } /** * Ask FLuxBench to encrypt message - * @param {object} message message object with information to be decrypted. + * @param {string} message JSON object with information to be encrypted. */ async function encryptMessage(message) { - const rpccall = 'encryptmessage'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('encryptmessage', message); } /** * Ask FLuxBench to get public key to encrypt enterprise content - * @param {object} message message object with the key. + * @param {string} message JSON object with the app and its owner. */ async function getPublicKey(message) { - const rpccall = 'getpublickey'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('getpublickey', message); } // == Control == diff --git a/tests/unit/benchmarkService.test.js b/tests/unit/benchmarkService.test.js index 4e34cd7a63..3c5a991165 100644 --- a/tests/unit/benchmarkService.test.js +++ b/tests/unit/benchmarkService.test.js @@ -125,6 +125,87 @@ describe('benchmarkService tests', () => { }); }); + describe('app request tests', () => { + const appCalls = [ + ['getPublicKey', 'getpublickey'], + ['decryptRSAMessage', 'decryptrsamessage'], + ['encryptMessage', 'encryptmessage'], + ['decryptMessage', 'decryptmessage'], + ]; + const validRequest = { + fluxID: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', + appName: 'my-app1', + message: 'aGVsbG8=', + blockHeight: 2000000, + }; + const refused = { + status: 'error', + data: { code: undefined, name: undefined, message: 'Invalid app name or owner' }, + }; + let benchmarkStub; + + beforeEach(() => { + sinon.stub(fs, 'stat').resolves(true); + benchmarkStub = sinon.stub(fluxRpc.FluxRpc.prototype, 'run').resolves('called'); + }); + + afterEach(() => { + sinon.restore(); + }); + + appCalls.forEach(([fn, rpc]) => { + it(`${fn} should send a request with a valid app name and owner`, async () => { + const message = JSON.stringify(validRequest); + + const result = await benchmarkService[fn](message); + + expect(result).to.eql({ status: 'success', data: 'called' }); + sinon.assert.calledOnceWithExactly(benchmarkStub, rpc, { params: [message] }); + }); + + it(`${fn} should send a request whose owner is an Ethereum address`, async () => { + const message = JSON.stringify({ ...validRequest, fluxID: `0x${'aB3'.repeat(13)}c` }); + + const result = await benchmarkService[fn](message); + + expect(result.status).to.equal('success'); + sinon.assert.calledOnce(benchmarkStub); + }); + + [ + ['an app name with an underscore', { appName: 'my_app' }], + ['an app name with a space', { appName: 'my app' }], + ['an app name starting with a hyphen', { appName: '-myapp' }], + ['an app name longer than 63 characters', { appName: 'a'.repeat(64) }], + ['an app name that is not a string', { appName: ['myapp'] }], + ['no app name', { appName: undefined }], + ['an owner that is not a Flux ID or Ethereum address', { fluxID: 'my-owner' }], + ['an owner with a bad checksum', { fluxID: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uD' }], + ['an Ethereum owner that is too long', { fluxID: `0x${'a'.repeat(41)}` }], + ['an owner that is not a string', { fluxID: { id: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' } }], + ['no owner', { fluxID: undefined }], + ].forEach(([description, change]) => { + it(`${fn} should refuse a request with ${description}`, async () => { + const message = JSON.stringify({ ...validRequest, ...change }); + + const result = await benchmarkService[fn](message); + + expect(result).to.eql(refused); + sinon.assert.notCalled(benchmarkStub); + }); + }); + + ['not json', 'null', '"myapp"', '[]'].forEach((message) => { + it(`${fn} should refuse ${message} as a request`, async () => { + const result = await benchmarkService[fn](message); + + expect(result).to.eql(refused); + sinon.assert.notCalled(benchmarkStub); + }); + }); + }); + }); + describe('getStatus tests', () => { let benchmarkStub; From 6d87a31589ad199ca5f417164ec8fb10a6e3ab25 Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 07:11:42 +0000 Subject: [PATCH 364/369] fix(apps): validate app name and owner where a spec arrives utils/appIdentity holds the rule: isValidAppNameAndOwner and verifyAppNameAndOwner, from APP_NAME_REGEX, MAX_APP_NAME_LENGTH and signatureVerifier.isValidSigningIdentity. The price, verify, register, update and public-key routes check every spec's name and owner before reading it. Peer messages and history sync check a spec's name and owner before reading it; history sync stores a message that fails the check, as it does any message it cannot read. Co-Authored-By: Claude Opus 5.5 --- .../services/appDatabase/registryManager.js | 3 + .../appLifecycle/advancedWorkflows.js | 2 + .../appMessaging/appHashSyncService.js | 3 + .../appMessaging/cryptographicKeys.js | 2 + .../src/services/appMessaging/messageStore.js | 2 + .../services/appRequirements/appValidator.js | 5 + ZelBack/src/services/benchmarkService.js | 12 +- ZelBack/src/services/utils/appIdentity.js | 32 +++++ ZelBack/src/services/utils/appSpecHelpers.js | 2 + tests/unit/appHashSyncService.test.js | 112 +++++++++++++++++- tests/unit/appIdentity.test.js | 55 +++++++++ tests/unit/appSpecHelpers.test.js | 22 ++++ tests/unit/appValidator.test.js | 52 ++++++++ tests/unit/benchmarkService.test.js | 22 +--- tests/unit/cryptographicKeys.test.js | 65 ++++++---- tests/unit/messageStore.test.js | 46 ++++++- tests/unit/registryManager.test.js | 51 ++++++++ tests/unit/updateAppGlobaly.test.js | 19 +++ 18 files changed, 452 insertions(+), 55 deletions(-) create mode 100644 ZelBack/src/services/utils/appIdentity.js create mode 100644 tests/unit/appIdentity.test.js diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index a9cff1c4b0..fd1ada9ac5 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -10,6 +10,7 @@ const globalState = require('../utils/globalState'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../utils/appSyncEvents'); // Removed appsService to avoid circular dependency - will use dynamic require where needed const { checkAndDecryptAppSpecs, encryptEnterpriseFromSession } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const { specificationFormatter, updateToLatestAppSpecifications } = require('../utils/appUtilities'); const placementFeasibility = require('../appPlacement/placementFeasibility'); const { withRegistryWrite } = require('./registryWriteLock'); @@ -2148,6 +2149,8 @@ async function registerAppGlobalyApi(req, res) { } const daemonHeight = syncStatus.data.height; + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecification, { diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index a761b1e7d7..c7b3b274e2 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -37,6 +37,7 @@ const { const { specificationFormatter } = require('../utils/appSpecHelpers'); const { compareInstanceSeniority } = require('../utils/instanceOrdering'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const volumeService = require('../utils/volumeService'); const mountParser = require('../utils/mountParser'); const appReconciler = require('../appMonitoring/appReconciler'); @@ -3563,6 +3564,7 @@ async function updateAppGlobaly(params) { const daemonHeight = syncStatus.data.height; const appSpecObj = serviceHelper.ensureObject(appSpecification); + verifyAppNameAndOwner(appSpecObj.name, appSpecObj.owner); const appSpecDecrypted = await checkAndDecryptAppSpecs(appSpecObj, { daemonHeight }); const appSpecFormatted = specificationFormatter(appSpecDecrypted); diff --git a/ZelBack/src/services/appMessaging/appHashSyncService.js b/ZelBack/src/services/appMessaging/appHashSyncService.js index 0d0860e3b2..9a727db4df 100644 --- a/ZelBack/src/services/appMessaging/appHashSyncService.js +++ b/ZelBack/src/services/appMessaging/appHashSyncService.js @@ -9,6 +9,7 @@ const messageVerifier = require('./messageVerifier'); const appValidator = require('../appRequirements/appValidator'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const { serialiseAndSignFluxBroadcast } = require('../utils/fluxBroadcastHelper'); const { peerManager } = require('../utils/peerState'); @@ -374,6 +375,7 @@ async function processMessages(messages, records, onProgress) { if (appSpecFormatted.version >= 8 && appSpecFormatted.enterprise) { try { + verifyAppNameAndOwner(appSpecFormatted.name, appSpecFormatted.owner); const decrypted = await checkAndDecryptAppSpecs( appSpecFormatted, { daemonHeight: height, owner: appSpecFormatted.owner }, @@ -414,6 +416,7 @@ async function processMessages(messages, records, onProgress) { let prevSpecsForVerification = prevSpecs; if (prevSpecs.version >= 8 && prevSpecs.enterprise) { try { + verifyAppNameAndOwner(prevSpecs.name, prevSpecs.owner); const decrypted = await checkAndDecryptAppSpecs( prevSpecs, { daemonHeight: prevMsg.height, owner: prevSpecs.owner }, ); diff --git a/ZelBack/src/services/appMessaging/cryptographicKeys.js b/ZelBack/src/services/appMessaging/cryptographicKeys.js index a35775536a..9e9fc5ad09 100644 --- a/ZelBack/src/services/appMessaging/cryptographicKeys.js +++ b/ZelBack/src/services/appMessaging/cryptographicKeys.js @@ -6,6 +6,7 @@ const benchmarkService = require('../benchmarkService'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const log = require('../../lib/log'); const { Privilege, authOf } = require('../utils/privileges'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); // Check if running on Arcane OS const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -67,6 +68,7 @@ async function getPublicKey(req, res) { if (!appSpecification.owner || !appSpecification.name) { throw new Error('Input parameters missing.'); } + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); if (!syncStatus.data.synced) { throw new Error('Daemon not yet synced.'); diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index 3b671a6569..fa09fc8217 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -9,6 +9,7 @@ const registryManager = require('../appDatabase/registryManager'); const { socketAddressesMatch } = require('../utils/socketAddressUtils'); // const advancedWorkflows = require('../appLifecycle/advancedWorkflows'); // Moved to dynamic require to avoid circular dependency const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const globalState = require('../utils/globalState'); const { appRunningEntries } = require('../utils/appRunningEntries'); const fluxEventBus = require('../utils/fluxEventBus'); @@ -168,6 +169,7 @@ async function storeAppTemporaryMessage(message, options = {}) { // eslint-disable-next-line global-require const fluxService = require('../fluxService'); if (await fluxService.isSystemSecure()) { + verifyAppNameAndOwner(appSpecFormatted.name, appSpecFormatted.owner); // eslint-disable-next-line no-use-before-define const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecFormatted, diff --git a/ZelBack/src/services/appRequirements/appValidator.js b/ZelBack/src/services/appRequirements/appValidator.js index 933ae02f40..87d4ae53de 100644 --- a/ZelBack/src/services/appRequirements/appValidator.js +++ b/ZelBack/src/services/appRequirements/appValidator.js @@ -20,6 +20,7 @@ const { } = require('../utils/appConstants'); const { specificationFormatter, findCommonArchitectures } = require('../utils/appUtilities'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const placementFeasibility = require('../appPlacement/placementFeasibility'); const enterpriseConfig = require('../utils/enterpriseConfig'); const portManager = require('../appNetwork/portManager'); @@ -1510,6 +1511,8 @@ async function verifyAppRegistrationParameters(req, res) { appSpecification.version >= 8 && appSpecification.enterprise, ); + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + // Decrypt enterprise specifications if needed const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecification, @@ -1586,6 +1589,8 @@ async function validateAppUpdate(appSpecification) { appSpecification.version >= 8 && appSpecification.enterprise, ); + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + const decryptedSpecs = await checkAndDecryptAppSpecs(appSpecification, { daemonHeight }); const appSpecFormatted = specificationFormatter(decryptedSpecs); diff --git a/ZelBack/src/services/benchmarkService.js b/ZelBack/src/services/benchmarkService.js index 67e15c07b0..c9bdb320a2 100644 --- a/ZelBack/src/services/benchmarkService.js +++ b/ZelBack/src/services/benchmarkService.js @@ -11,9 +11,8 @@ const generalService = require('./generalService'); const upnpService = require('./upnpService'); const fluxRpc = require('./utils/fluxRpc'); const dbHelper = require('./dbHelper'); -const signatureVerifier = require('./signatureVerifier'); const { Privilege, authOf } = require('./utils/privileges'); -const { APP_NAME_REGEX, MAX_APP_NAME_LENGTH } = require('./utils/appConstants'); +const { isValidAppNameAndOwner } = require('./utils/appIdentity'); // eslint-disable-next-line no-unused-vars const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -282,8 +281,7 @@ async function signFluxTransactionPost(req, res) { } /** - * Whether a request names a valid app and a valid owner: an app name - * as a spec allows it, and an owner that can sign (a Flux ID or an Ethereum address). + * Whether a request names a well-formed app and owner. * @param {string} message JSON object with `appName` and `fluxID`. * @returns {boolean} */ @@ -296,11 +294,7 @@ function isValidAppRequest(message) { } if (!request || typeof request !== 'object') return false; - const { appName, fluxID } = request; - return typeof appName === 'string' - && appName.length <= MAX_APP_NAME_LENGTH - && APP_NAME_REGEX.test(appName) - && signatureVerifier.isValidSigningIdentity(fluxID); + return isValidAppNameAndOwner(request.appName, request.fluxID); } /** diff --git a/ZelBack/src/services/utils/appIdentity.js b/ZelBack/src/services/utils/appIdentity.js new file mode 100644 index 0000000000..a761059185 --- /dev/null +++ b/ZelBack/src/services/utils/appIdentity.js @@ -0,0 +1,32 @@ +const signatureVerifier = require('../signatureVerifier'); +const { APP_NAME_REGEX, MAX_APP_NAME_LENGTH } = require('./appConstants'); + +/** + * Whether an app name and owner are well formed: a name as a spec allows it, + * and an owner that can sign (a Flux ID or an Ethereum address). + * @param {*} name App name. + * @param {*} owner App owner. + * @returns {boolean} + */ +function isValidAppNameAndOwner(name, owner) { + return typeof name === 'string' + && name.length <= MAX_APP_NAME_LENGTH + && APP_NAME_REGEX.test(name) + && signatureVerifier.isValidSigningIdentity(owner); +} + +/** + * Throws unless the app name and owner are well formed. + * @param {*} name App name. + * @param {*} owner App owner. + */ +function verifyAppNameAndOwner(name, owner) { + if (!isValidAppNameAndOwner(name, owner)) { + throw new Error('Invalid app name or owner'); + } +} + +module.exports = { + isValidAppNameAndOwner, + verifyAppNameAndOwner, +}; diff --git a/ZelBack/src/services/utils/appSpecHelpers.js b/ZelBack/src/services/utils/appSpecHelpers.js index cbc5b605d1..8a891b2e6f 100644 --- a/ZelBack/src/services/utils/appSpecHelpers.js +++ b/ZelBack/src/services/utils/appSpecHelpers.js @@ -5,6 +5,7 @@ const messageHelper = require('../messageHelper'); const serviceHelper = require('../serviceHelper'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const { checkAndDecryptAppSpecs } = require('./enterpriseHelper'); +const { verifyAppNameAndOwner } = require('./appIdentity'); const { appPricePerMonth } = require('./appUtilities'); const { specificationFormatter } = require('./appUtilities'); const { getChainParamsPriceUpdates } = require('./chainUtilities'); @@ -325,6 +326,7 @@ async function getAppFiatAndFluxPrice(req, res) { throw new Error('Daemon not yet synced.'); } const daemonHeight = syncStatus.data.height; + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); appSpecification = await checkAndDecryptAppSpecs(appSpecification, { daemonHeight }); const appSpecFormatted = specificationFormatter(appSpecification); diff --git a/tests/unit/appHashSyncService.test.js b/tests/unit/appHashSyncService.test.js index ac1a85d307..6a6f3a79f3 100644 --- a/tests/unit/appHashSyncService.test.js +++ b/tests/unit/appHashSyncService.test.js @@ -1254,12 +1254,59 @@ describe('appHashSyncService tests', () => { expect(inserted.valueSat).to.equal(105); }); + describe('an enterprise registration', () => { + const syncRegistration = async (change) => { + const bulkMessages = [{ + type: 'fluxappregister', version: 1, hash: 'hash5', timestamp: Date.now(), signature: 'sig5', + appSpecifications: { + name: 'entapp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'blob', ...change, + }, + valueSat: 1e8, txid: 'tx5', height: 1005, + }]; + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1000 + i, value: 100 + i, message: false, + })); + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + if (getMissingCalls === 1) return Promise.resolve(manyMissing); + return Promise.resolve([]); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse(bulkMessages)); + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + + await localModule.syncMissingHashes(); + }; + + it('should decrypt a registration with a valid app name and owner', async () => { + await syncRegistration({}); + + sinon.assert.calledOnce(localCheckAndDecryptStub); + expect(localCollectionStub.insertMany.firstCall.args[0][0].hash).to.equal('hash5'); + }); + + [ + ['an invalid app name', { name: 'ent_app' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should store a registration with ${description} without decrypting it`, async () => { + await syncRegistration(change); + + sinon.assert.notCalled(localCheckAndDecryptStub); + expect(localCollectionStub.insertMany.firstCall.args[0][0].hash).to.equal('hash5'); + }); + }); + }); + it('should decrypt prevSpec for enterprise v8 updates before signature verification', async () => { const bulkMessages = [ { type: 'fluxappupdate', version: 8, hash: 'hash1', timestamp: Date.now(), signature: 'sig1', - appSpecifications: { name: 'enterpriseApp', version: 8, owner: 'owner1', enterprise: 'encBlob' }, + appSpecifications: { name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'encBlob' }, valueSat: 1e8, txid: 'tx1', height: 2000, }, ]; @@ -1286,7 +1333,7 @@ describe('appHashSyncService tests', () => { // Previous spec is enterprise v8 with encrypted data const prevSpec = { - name: 'enterpriseApp', version: 8, owner: 'owner1', enterprise: 'prevEncBlob', + name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'prevEncBlob', }; localCollectionStub.find.returns({ project: sinon.stub().returns({ @@ -1302,7 +1349,7 @@ describe('appHashSyncService tests', () => { }); const decryptedPrevSpec = { - name: 'enterpriseApp', version: 8, owner: 'owner1', + name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', compose: [{ name: 'comp1', repotag: 'repo/tag' }], }; @@ -1325,7 +1372,64 @@ describe('appHashSyncService tests', () => { const verifyCall = localMessageVerifierStub.verifyAppMessageUpdateSignature.firstCall; expect(verifyCall).to.not.be.null; // The owner param (6th arg) should be from the decrypted prevSpec - expect(verifyCall.args[5]).to.equal('owner1'); + expect(verifyCall.args[5]).to.equal('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC'); + }); + + it('should verify an update against the stored prevSpec when its owner is invalid, without decrypting it', async () => { + const bulkMessages = [ + { + type: 'fluxappupdate', version: 8, hash: 'hash1', timestamp: Date.now(), + signature: 'sig1', + appSpecifications: { name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'encBlob' }, + valueSat: 1e8, txid: 'tx1', height: 2000, + }, + ]; + + // Generate > 500 missing hashes to trigger bulk fetch path + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1999 + i, value: 1e8, message: false, + })); + + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + if (getMissingCalls === 1) return Promise.resolve(manyMissing); + return Promise.resolve([]); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) { + return Promise.resolve(makeStreamResponse(bulkMessages)); + } + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + + // Previous spec is enterprise v8 with encrypted data + const prevSpec = { + name: 'enterpriseApp', version: 8, owner: 'my-owner', enterprise: 'prevEncBlob', + }; + localCollectionStub.find.returns({ + project: sinon.stub().returns({ + sort: sinon.stub().returns({ + toArray: sinon.stub().resolves([ + { + type: 'fluxappregister', hash: 'hash0', height: 1999, + appSpecifications: prevSpec, + }, + ]), + }), + }), + }); + + localMessageVerifierStub.verifyAppMessageUpdateSignature.resolves(true); + + await localModule.syncMissingHashes(); + + // only the update itself is decrypted + sinon.assert.calledOnce(localCheckAndDecryptStub); + const verifyCall = localMessageVerifierStub.verifyAppMessageUpdateSignature.firstCall; + expect(verifyCall.args[5]).to.equal('my-owner'); }); }); diff --git a/tests/unit/appIdentity.test.js b/tests/unit/appIdentity.test.js new file mode 100644 index 0000000000..236f622d6e --- /dev/null +++ b/tests/unit/appIdentity.test.js @@ -0,0 +1,55 @@ +const { expect } = require('chai'); + +const { isValidAppNameAndOwner, verifyAppNameAndOwner } = require('../../ZelBack/src/services/utils/appIdentity'); + +describe('appIdentity tests', () => { + const fluxId = '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC'; + const ethAddress = `0x${'aB3'.repeat(13)}c`; + + describe('isValidAppNameAndOwner tests', () => { + [ + ['a Flux ID owner', 'my-app1', fluxId], + ['an Ethereum owner', 'my-app1', ethAddress], + ['a 63 character name', 'a'.repeat(63), fluxId], + ['a one character name', 'a', fluxId], + ].forEach(([description, name, owner]) => { + it(`should accept ${description}`, () => { + expect(isValidAppNameAndOwner(name, owner)).to.equal(true); + }); + }); + + [ + ['a name with an underscore', 'my_app', fluxId], + ['a name with a space', 'my app', fluxId], + ['a name starting with a hyphen', '-myapp', fluxId], + ['a name ending with a hyphen', 'myapp-', fluxId], + ['a name longer than 63 characters', 'a'.repeat(64), fluxId], + ['an empty name', '', fluxId], + ['a name that is not a string', ['myapp'], fluxId], + ['no name', undefined, fluxId], + ['an owner that is not a Flux ID or Ethereum address', 'myapp', 'my-owner'], + ['an owner with a bad checksum', 'myapp', '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uD'], + ['an Ethereum owner that is too long', 'myapp', `0x${'a'.repeat(41)}`], + ['an owner that is not a string', 'myapp', { id: fluxId }], + ['no owner', 'myapp', undefined], + ].forEach(([description, name, owner]) => { + it(`should refuse ${description}`, () => { + expect(isValidAppNameAndOwner(name, owner)).to.equal(false); + }); + }); + }); + + describe('verifyAppNameAndOwner tests', () => { + it('should return for a valid name and owner', () => { + expect(() => verifyAppNameAndOwner('myapp', fluxId)).to.not.throw(); + }); + + it('should throw for an invalid name', () => { + expect(() => verifyAppNameAndOwner('my_app', fluxId)).to.throw('Invalid app name or owner'); + }); + + it('should throw for an invalid owner', () => { + expect(() => verifyAppNameAndOwner('myapp', 'my-owner')).to.throw('Invalid app name or owner'); + }); + }); +}); diff --git a/tests/unit/appSpecHelpers.test.js b/tests/unit/appSpecHelpers.test.js index 2a41564de8..9c700935e6 100644 --- a/tests/unit/appSpecHelpers.test.js +++ b/tests/unit/appSpecHelpers.test.js @@ -868,6 +868,28 @@ describe('appSpecHelpers tests', () => { const response = await quote(buildSpec('g:/data', { priceUSD: 5.1 })); expect(response.data.usd).to.equal(5.1); }); + + [ + ['an invalid app name', { name: 'Price_Test' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should refuse a spec with ${description}`, async () => { + const response = await quote(buildSpec('/data', change)); + expect(response.status).to.equal('error'); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + + it(`should refuse an enterprise spec with ${description} before decrypting it`, async () => { + const response = await quote(buildSpec('/data', { version: 8, enterprise: 'blob', ...change })); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + }); + + it('should decrypt an enterprise spec with a valid app name and owner', async () => { + // Off ArcaneOS the decrypt refuses, which shows the spec reached it. + const response = await quote(buildSpec('/data', { version: 8, enterprise: 'blob' })); + expect(response.data.message).to.equal('Application Specifications can only be validated on a node running Arcane OS.'); + }); }); describe('module exports tests', () => { diff --git a/tests/unit/appValidator.test.js b/tests/unit/appValidator.test.js index 7d13880d8b..37eb1a2b18 100644 --- a/tests/unit/appValidator.test.js +++ b/tests/unit/appValidator.test.js @@ -1,6 +1,9 @@ +const { EventEmitter } = require('events'); const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); +const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/daemonServiceMiscRpcs'); +const appValidatorService = require('../../ZelBack/src/services/appRequirements/appValidator'); describe('appValidator tests', () => { let appValidator; @@ -1056,3 +1059,52 @@ describe('appValidator tests', () => { }); }); }); + +describe('appValidator spec verification route tests', () => { + const spec = { + version: 8, + name: 'verifytest', + description: 'verify test', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', + enterprise: 'blob', + }; + + const post = (handler, body) => new Promise((resolve) => { + const req = new EventEmitter(); + const res = { json: (response) => resolve(response) }; + handler(req, res); + req.emit('data', JSON.stringify(body)); + req.emit('end'); + }); + + beforeEach(() => { + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 2500000 } }); + }); + + afterEach(() => { + sinon.restore(); + }); + + [ + ['verifyAppRegistrationParameters', appValidatorService.verifyAppRegistrationParameters], + ['verifyAppUpdateApi', appValidatorService.verifyAppUpdateApi], + ].forEach(([route, handler]) => { + [ + ['an invalid app name', { name: 'verify_test' }], + ['an invalid owner', { owner: 'my-owner' }], + ['an invalid app name in a spec that is not enterprise', { version: 4, enterprise: undefined, name: 'verify_test' }], + ].forEach(([description, change]) => { + it(`${route} should refuse a spec with ${description}`, async () => { + const response = await post(handler, { ...spec, ...change }); + expect(response.status).to.equal('error'); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + }); + + it(`${route} should decrypt a spec with a valid app name and owner`, async () => { + // Off ArcaneOS the decrypt refuses, which shows the spec reached it. + const response = await post(handler, spec); + expect(response.data.message).to.equal('Application Specifications can only be validated on a node running Arcane OS.'); + }); + }); +}); diff --git a/tests/unit/benchmarkService.test.js b/tests/unit/benchmarkService.test.js index 3c5a991165..55a57aac11 100644 --- a/tests/unit/benchmarkService.test.js +++ b/tests/unit/benchmarkService.test.js @@ -163,27 +163,9 @@ describe('benchmarkService tests', () => { sinon.assert.calledOnceWithExactly(benchmarkStub, rpc, { params: [message] }); }); - it(`${fn} should send a request whose owner is an Ethereum address`, async () => { - const message = JSON.stringify({ ...validRequest, fluxID: `0x${'aB3'.repeat(13)}c` }); - - const result = await benchmarkService[fn](message); - - expect(result.status).to.equal('success'); - sinon.assert.calledOnce(benchmarkStub); - }); - [ - ['an app name with an underscore', { appName: 'my_app' }], - ['an app name with a space', { appName: 'my app' }], - ['an app name starting with a hyphen', { appName: '-myapp' }], - ['an app name longer than 63 characters', { appName: 'a'.repeat(64) }], - ['an app name that is not a string', { appName: ['myapp'] }], - ['no app name', { appName: undefined }], - ['an owner that is not a Flux ID or Ethereum address', { fluxID: 'my-owner' }], - ['an owner with a bad checksum', { fluxID: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uD' }], - ['an Ethereum owner that is too long', { fluxID: `0x${'a'.repeat(41)}` }], - ['an owner that is not a string', { fluxID: { id: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' } }], - ['no owner', { fluxID: undefined }], + ['an invalid app name', { appName: 'my_app' }], + ['an invalid owner', { fluxID: 'my-owner' }], ].forEach(([description, change]) => { it(`${fn} should refuse a request with ${description}`, async () => { const message = JSON.stringify({ ...validRequest, ...change }); diff --git a/tests/unit/cryptographicKeys.test.js b/tests/unit/cryptographicKeys.test.js index 17c04cb6f8..5b203f1137 100644 --- a/tests/unit/cryptographicKeys.test.js +++ b/tests/unit/cryptographicKeys.test.js @@ -64,7 +64,7 @@ describe('cryptographicKeys tests', () => { reloadModule(); try { - await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect.fail('Should have thrown error'); } catch (error) { expect(error.message).to.include('Arcane OS'); @@ -85,7 +85,7 @@ describe('cryptographicKeys tests', () => { }), }); - const result = await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + const result = await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect(result).to.equal(mockPublicKey); }); @@ -100,7 +100,7 @@ describe('cryptographicKeys tests', () => { }); try { - await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect.fail('Should have thrown error'); } catch (error) { expect(error.message).to.include('Error getting public key'); @@ -120,7 +120,7 @@ describe('cryptographicKeys tests', () => { }); try { - await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect.fail('Should have thrown error'); } catch (error) { expect(error.message).to.include('Error getting public key to encrypt app enterprise content from SAS'); @@ -139,7 +139,7 @@ describe('cryptographicKeys tests', () => { }), }); - const fluxID = '1ABC123XYZ'; + const fluxID = '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uCXYZ'; const appName = 'MyTestApp'; const blockHeight = 123456; @@ -162,7 +162,7 @@ describe('cryptographicKeys tests', () => { }); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -195,13 +195,13 @@ describe('cryptographicKeys tests', () => { process.env.FLUXOS_PATH = '/some/path'; reloadModule(); sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); - sinon.stub(serviceHelper, 'ensureObject').returns({ owner: '1ABC123' }); // Missing name + sinon.stub(serviceHelper, 'ensureObject').returns({ owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' }); // Missing name sinon.stub(messageHelper, 'createErrorMessage').callsFake((msg) => ({ status: 'error', data: { message: msg }, })); - const appSpec = JSON.stringify({ owner: '1ABC123' }); + const appSpec = JSON.stringify({ owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' }); await callGetPublicKey(req, res, appSpec); @@ -215,7 +215,7 @@ describe('cryptographicKeys tests', () => { reloadModule(); sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -228,7 +228,7 @@ describe('cryptographicKeys tests', () => { })); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -245,7 +245,7 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -265,7 +265,7 @@ describe('cryptographicKeys tests', () => { })); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -276,6 +276,29 @@ describe('cryptographicKeys tests', () => { expect(res.json.firstCall.args[0].data).to.equal(mockPublicKey); }); + [ + ['an invalid app name', { owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'Test_App' }], + ['an invalid owner', { owner: 'my-owner', name: 'TestApp' }], + ].forEach(([description, request]) => { + it(`should refuse a request with ${description}`, async () => { + process.env.FLUXOS_PATH = '/some/path'; + reloadModule(); + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ + status: 'success', + data: { synced: true, height: 2000 }, + }); + const publicKeyCall = sinon.stub(benchmarkService, 'getPublicKey').resolves({ status: 'success', data: '{}' }); + + await callGetPublicKey(req, res, JSON.stringify(request)); + + sinon.assert.calledOnce(res.json); + expect(res.json.firstCall.args[0].status).to.equal('error'); + expect(res.json.firstCall.args[0].data.message).to.equal('Invalid app name or owner'); + sinon.assert.notCalled(publicKeyCall); + }); + }); + it('should handle request body in chunks', async () => { process.env.FLUXOS_PATH = '/some/path'; reloadModule(); @@ -283,9 +306,9 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); const ensureObjectStub = sinon.stub(serviceHelper, 'ensureObject'); - ensureObjectStub.onFirstCall().returns('{"owner":"1ABC123","name":"TestApp"}'); + ensureObjectStub.onFirstCall().returns('{"owner":"1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC","name":"TestApp"}'); ensureObjectStub.onSecondCall().returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -305,7 +328,7 @@ describe('cryptographicKeys tests', () => { })); // Send request body in multiple chunks - await callGetPublicKey(req, res, ['{"owner":"1ABC', '123","name":"T', 'estApp"}']); + await callGetPublicKey(req, res, ['{"owner":"1CbErtneaX2QV', 'yUfwU7JGB7VzvPgrgc3uC","name":"T', 'estApp"}']); sinon.assert.calledOnce(res.json); expect(res.json.firstCall.args[0].status).to.equal('success'); @@ -316,7 +339,7 @@ describe('cryptographicKeys tests', () => { reloadModule(); const verifyStub = sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -336,7 +359,7 @@ describe('cryptographicKeys tests', () => { }); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -352,7 +375,7 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -372,7 +395,7 @@ describe('cryptographicKeys tests', () => { }); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -389,7 +412,7 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -406,7 +429,7 @@ describe('cryptographicKeys tests', () => { })); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); diff --git a/tests/unit/messageStore.test.js b/tests/unit/messageStore.test.js index 0a26089fe7..d31ecf1fb4 100644 --- a/tests/unit/messageStore.test.js +++ b/tests/unit/messageStore.test.js @@ -1,9 +1,11 @@ const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); +const fluxService = require('../../ZelBack/src/services/fluxService'); describe('messageStore tests', () => { let messageStore; + let checkAndDecryptStub; let dbHelperStub; let serviceHelperStub; let messageVerifierStub; @@ -12,6 +14,7 @@ describe('messageStore tests', () => { beforeEach(() => { // Stubs + checkAndDecryptStub = sinon.stub().resolves({}); dbHelperStub = { databaseConnection: sinon.stub(), findInDatabase: sinon.stub(), @@ -82,7 +85,7 @@ describe('messageStore tests', () => { validateApplicationUpdateCompatibility: sinon.stub().resolves(), }, '../utils/enterpriseHelper': { - checkAndDecryptAppSpecs: sinon.stub().resolves({}), + checkAndDecryptAppSpecs: checkAndDecryptStub, }, '../utils/appConstants': { globalAppsMessages: 'appsMessages', @@ -213,6 +216,47 @@ describe('messageStore tests', () => { } }); + describe('enterprise registration', () => { + const enterpriseMessage = (change) => ({ + type: 'fluxappregister', + version: 1, + appSpecifications: { + name: 'entapp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'blob', ...change, + }, + hash: 'hash123', + timestamp: Date.now(), + signature: 'sig123', + }); + + beforeEach(() => { + sinon.stub(fluxService, 'isSystemSecure').resolves(true); + messageVerifierStub.checkAppMessageExistence.resolves(null); + messageVerifierStub.checkAppTemporaryMessageExistence.resolves(null); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findOneInDatabase.resolves(null); + }); + + [ + ['an invalid app name', { name: 'ent_app' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should refuse a message with ${description} without decrypting it`, async () => { + await messageStore.storeAppTemporaryMessage(enterpriseMessage(change)).then( + () => expect.fail('should have refused'), + (error) => expect(error.message).to.equal('Invalid app name or owner'), + ); + sinon.assert.notCalled(checkAndDecryptStub); + sinon.assert.notCalled(dbHelperStub.insertOneToDatabase); + }); + }); + + it('should decrypt a message with a valid app name and owner', async () => { + // The stubs end the message after the decrypt; only reaching it matters here. + await messageStore.storeAppTemporaryMessage(enterpriseMessage({})).catch(() => {}); + sinon.assert.calledOnce(checkAndDecryptStub); + }); + }); + it('should not enforce version upgrade policy (enforced at API layer)', async () => { const message = { type: 'fluxappupdate', diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 7ba91e4659..53e1dd9e7a 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1,3 +1,4 @@ +const { EventEmitter } = require('events'); const { expect } = require('chai'); const sinon = require('sinon'); const config = require('config'); @@ -15,6 +16,7 @@ const verificationHelper = require('../../ZelBack/src/services/verificationHelpe const globalState = require('../../ZelBack/src/services/utils/globalState'); const localRemovalQueue = require('../../ZelBack/src/services/appLifecycle/localRemovalQueue'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); +const { peerManager } = require('../../ZelBack/src/services/utils/peerState'); const { requireMongo } = require('./dbTestHelper'); describe('registryManager tests', () => { @@ -2070,3 +2072,52 @@ describe('registryManager tests', () => { }); }); }); + +describe('registryManager registerAppGlobalyApi tests', () => { + const spec = { + version: 8, + name: 'registertest', + description: 'register test', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', + enterprise: 'blob', + }; + + const register = (appSpecification) => new Promise((resolve) => { + const req = new EventEmitter(); + const res = { json: (response) => resolve(response) }; + registryManager.registerAppGlobalyApi(req, res); + req.emit('data', JSON.stringify({ + appSpecification, timestamp: Date.now(), signature: 'sig', type: 'fluxappregister', version: 1, + })); + req.emit('end'); + }); + + beforeEach(() => { + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + sinon.stub(peerManager, 'outboundCount').get(() => 100); + sinon.stub(peerManager, 'inboundCount').get(() => 100); + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 2500000 } }); + }); + + afterEach(() => { + sinon.restore(); + }); + + [ + ['an invalid app name', { name: 'register_test' }], + ['an invalid owner', { owner: 'my-owner' }], + ['an invalid app name in a spec that is not enterprise', { version: 4, enterprise: undefined, name: 'register_test' }], + ].forEach(([description, change]) => { + it(`should refuse a spec with ${description}`, async () => { + const response = await register({ ...spec, ...change }); + expect(response.status).to.equal('error'); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + }); + + it('should decrypt a spec with a valid app name and owner', async () => { + // Off ArcaneOS the decrypt refuses, which shows the spec reached it. + const response = await register(spec); + expect(response.data.message).to.equal('Application Specifications can only be validated on a node running Arcane OS.'); + }); +}); diff --git a/tests/unit/updateAppGlobaly.test.js b/tests/unit/updateAppGlobaly.test.js index 5e4a80b9c7..516f680ece 100644 --- a/tests/unit/updateAppGlobaly.test.js +++ b/tests/unit/updateAppGlobaly.test.js @@ -107,4 +107,23 @@ describe('updateAppGlobaly tests', () => { ); expect(signatureCheck.called).to.equal(false); }); + + [ + ['an invalid app name', { name: 'Submit_App' }], + ['an invalid owner', { owner: 'my-owner' }], + ['an invalid owner in an enterprise spec', { version: 8, enterprise: 'blob', owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should refuse an update with ${description} before reading the spec`, async () => { + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 3003100 } }); + + await advancedWorkflows.updateAppGlobaly({ + appSpecification: { ...spec, ...change }, timestamp: Date.now(), signature: 'sig', type: 'fluxappupdate', version: 1, + }).then( + () => expect.fail('should have refused'), + (error) => expect(error.message).to.equal('Invalid app name or owner'), + ); + sinon.assert.notCalled(appValidator.verifyAppSpecifications); + expect(signatureCheck.called).to.equal(false); + }); + }); }); From 7c84d6b7d68d43c58e01be51650b16885e242fd1 Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 07:15:51 +0000 Subject: [PATCH 365/369] refactor(apps): remove appValidator's unused registerAppGlobalyApi /apps/appregister is served by registryManager.registerAppGlobalyApi; the copy in appValidator had no caller. The imports only it used go with it. Co-Authored-By: Claude Opus 5.5 --- .../services/appRequirements/appValidator.js | 171 ------------------ 1 file changed, 171 deletions(-) diff --git a/ZelBack/src/services/appRequirements/appValidator.js b/ZelBack/src/services/appRequirements/appValidator.js index 87d4ae53de..52ae30c92c 100644 --- a/ZelBack/src/services/appRequirements/appValidator.js +++ b/ZelBack/src/services/appRequirements/appValidator.js @@ -2,12 +2,8 @@ const config = require('config'); const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const log = require('../../lib/log'); -const generalService = require('../generalService'); -const verificationHelper = require('../verificationHelper'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); -const fluxCommunicationMessagesSender = require('../fluxCommunicationMessagesSender'); const registryManager = require('../appDatabase/registryManager'); -const messageVerifier = require('../appMessaging/messageVerifier'); const signatureVerifier = require('../signatureVerifier'); const fluxStorage = require('../utils/fluxStorage'); const imageManager = require('../appSecurity/imageManager'); @@ -24,10 +20,6 @@ const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const placementFeasibility = require('../appPlacement/placementFeasibility'); const enterpriseConfig = require('../utils/enterpriseConfig'); const portManager = require('../appNetwork/portManager'); -const { peerManager } = require('../utils/peerState'); -const { Privilege, authOf } = require('../utils/privileges'); - -const isArcane = Boolean(process.env.FLUXOS_PATH); /** * Verify type correctness of application specification @@ -1670,168 +1662,6 @@ async function verifyAppUpdateApi(req, res) { }); } -/** - * Register application globally via API - * @param {object} req - Request object - * @param {object} res - Response object - * @returns {Promise} Registration result - */ -async function registerAppGlobalyApi(req, res) { - let body = ''; - req.on('data', (data) => { - body += data; - }); - req.on('end', async () => { - try { - const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - return; - } - - // first check if this node is available for application registration - if (peerManager.outboundCount < config.fluxapps.minOutgoing) { - throw new Error('Sorry, This Flux does not have enough outgoing peers for safe application registration'); - } - if (peerManager.inboundCount < config.fluxapps.minIncoming) { - throw new Error('Sorry, This Flux does not have enough incoming peers for safe application registration'); - } - - const processedBody = serviceHelper.ensureObject(body); - // Note. Actually signature, timestamp is not needed. But we require it only to verify that user indeed has access to the private key of the owner zelid. - // name and port HAVE to be unique for application. Check if they don't exist in global database - // first let's check if all fields are present and have proper format except tiered and tiered specifications and those can be omitted - let { appSpecification, timestamp, signature } = processedBody; - let messageType = processedBody.type; // determines how data is treated in the future - let typeVersion = processedBody.version; // further determines how data is treated in the future - - if (!appSpecification || !timestamp || !signature || !messageType || !typeVersion) { - throw new Error('Incomplete message received. Check if appSpecification, type, version, timestamp and signature are provided.'); - } - - if (messageType !== 'zelappregister' && messageType !== 'fluxappregister') { - throw new Error('Invalid type of message'); - } - - if (typeVersion !== 1) { - throw new Error('Invalid version of message'); - } - - appSpecification = serviceHelper.ensureObject(appSpecification); - timestamp = serviceHelper.ensureNumber(timestamp); - signature = serviceHelper.ensureString(signature); - messageType = serviceHelper.ensureString(messageType); - typeVersion = serviceHelper.ensureNumber(typeVersion); - - const timestampNow = Date.now(); - if (timestamp < timestampNow - 1000 * 3600) { - throw new Error('Message timestamp is over 1 hour old, not valid. Check if your computer clock is synced and restart the registration process.'); - } else if (timestamp > timestampNow + 1000 * 60 * 5) { - throw new Error('Message timestamp from future, not valid. Check if your computer clock is synced and restart the registration process.'); - } - - const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); - if (!syncStatus.data.synced) { - throw new Error('Daemon not yet synced.'); - } - - const daemonHeight = syncStatus.data.height; - - const appSpecDecrypted = await checkAndDecryptAppSpecs( - appSpecification, - { - daemonHeight, - owner: appSpecification.owner, - }, - ); - - const appSpecFormatted = specificationFormatter(appSpecDecrypted); - - // parameters are now proper format and assigned. Check for their validity, if they are within limits, have propper ports, repotag exists, string lengths, specs are ok - await verifyAppSpecifications(appSpecFormatted, daemonHeight, true); - - if (appSpecFormatted.version === 7 && appSpecFormatted.nodes.length > 0) { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecFormatted.compose) { - if (appComponent.secrets) { - // eslint-disable-next-line no-await-in-loop - await imageManager.checkAppSecrets(appSpecFormatted.name, appComponent, appSpecFormatted.owner, true); - } - } - } - - // check if name is not yet registered - await registryManager.checkApplicationRegistrationNameConflicts(appSpecFormatted); - - const isEnterprise = Boolean( - appSpecification.version >= 8 && appSpecification.enterprise, - ); - - const toVerify = isEnterprise - ? specificationFormatter(appSpecification) - : appSpecFormatted; - - // check if zelid owner is correct ( done in message verification ) - // if signature is not correct, then specifications are not correct type or bad message received. Respond with 'Received message is invalid'; - await messageVerifier.verifyAppMessageSignature(messageType, typeVersion, toVerify, timestamp, signature); - - if (isEnterprise) { - appSpecFormatted.contacts = []; - appSpecFormatted.compose = []; - } - - // if all ok, then sha256 hash of entire message = message + timestamp + signature. We are hashing all to have always unique value. - // If hashing just specificiations, if application goes back to previous specifications, it may pose some issues if we have indeed correct state - // We respond with a hash that is supposed to go to transaction. - const message = messageType + typeVersion + JSON.stringify(appSpecFormatted) + timestamp + signature; - const messageHASH = await generalService.messageHash(message); - - // now all is great. Store appSpecFormatted, timestamp, signature and hash in appsTemporaryMessages. with 1 hours expiration time. Broadcast this message to all outgoing connections. - const temporaryAppMessage = { // specification of temp message - type: messageType, - version: typeVersion, - appSpecifications: appSpecFormatted, - hash: messageHASH, - timestamp, - signature, - arcaneSender: isArcane, - }; - await fluxCommunicationMessagesSender.broadcastTemporaryAppMessage(temporaryAppMessage); - // above takes 2-3 seconds - await serviceHelper.delay(1200); // it takes receiving node at least 1 second to process the message. Add 1200 ms mas for processing - // this operations takes 2.5-3.5 seconds and is heavy, message gets verified again. - await messageVerifier.requestAppMessage(messageHASH); // this itself verifies that Peers received our message broadcast AND peers send us the message back. By peers sending the message back we finally store it to our temporary message storage and rebroadcast it again - // request app message is quite slow and from performance testing message will appear roughly 5 seconds after ask - await serviceHelper.delay(1200); // 1200 ms mas for processing - peer sends message back to us - // check temporary message storage - let tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(messageHASH); // Cumulus measurement: after roughly 8 seconds here - for (let i = 0; i < 20; i += 1) { // ask for up to 20 times - 10 seconds. Must have been processed by that time or it failed. Cumulus measurement: Approx 5-6 seconds - if (!tempMessage) { - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(500); - // eslint-disable-next-line no-await-in-loop - tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(messageHASH); - } - } - if (tempMessage && typeof tempMessage === 'object' && !Array.isArray(tempMessage)) { - const responseHash = messageHelper.createDataMessage(tempMessage.hash); - res.json(responseHash); // all ok - return; - } - throw new Error('Unable to register application on the network. Try again later.'); - } catch (error) { - log.warn(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - res.json(errorResponse); - } - }); -} - module.exports = { verifyTypeCorrectnessOfApp, verifyRestrictionCorrectnessOfApp, @@ -1842,5 +1672,4 @@ module.exports = { verifyAppRegistrationParameters, validateAppUpdate, verifyAppUpdateApi, - registerAppGlobalyApi, }; From 6e7a7a6261604908fadee7333ddcf1178665deac Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 08:30:39 +0000 Subject: [PATCH 366/369] fix(apps): scope the app name and owner check to enterprise specs The price, verify, register and update routes checked every spec's name and owner. Only an enterprise spec (version >= 8 with an enterprise field) is decrypted, so the check now runs only for those; a non-enterprise spec, a marketplace price request among them, is no longer refused by it. getpublickey keeps its check, as every request there is decrypted. Co-Authored-By: Claude Opus 4.8 --- .../services/appDatabase/registryManager.js | 4 +++- .../appLifecycle/advancedWorkflows.js | 4 +++- .../services/appRequirements/appValidator.js | 8 ++++++-- ZelBack/src/services/utils/appSpecHelpers.js | 4 +++- tests/unit/appSpecHelpers.test.js | 19 ++++++++++++++----- tests/unit/appValidator.test.js | 3 +-- tests/unit/registryManager.test.js | 3 +-- tests/unit/updateAppGlobaly.test.js | 13 +++++++++---- 8 files changed, 40 insertions(+), 18 deletions(-) diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index fd1ada9ac5..268acf896a 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -2149,7 +2149,9 @@ async function registerAppGlobalyApi(req, res) { } const daemonHeight = syncStatus.data.height; - verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + if (appSpecification.version >= 8 && appSpecification.enterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecification, diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index c7b3b274e2..54576b2a37 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -3564,7 +3564,9 @@ async function updateAppGlobaly(params) { const daemonHeight = syncStatus.data.height; const appSpecObj = serviceHelper.ensureObject(appSpecification); - verifyAppNameAndOwner(appSpecObj.name, appSpecObj.owner); + if (appSpecObj.version >= 8 && appSpecObj.enterprise) { + verifyAppNameAndOwner(appSpecObj.name, appSpecObj.owner); + } const appSpecDecrypted = await checkAndDecryptAppSpecs(appSpecObj, { daemonHeight }); const appSpecFormatted = specificationFormatter(appSpecDecrypted); diff --git a/ZelBack/src/services/appRequirements/appValidator.js b/ZelBack/src/services/appRequirements/appValidator.js index 52ae30c92c..8f8845b9c7 100644 --- a/ZelBack/src/services/appRequirements/appValidator.js +++ b/ZelBack/src/services/appRequirements/appValidator.js @@ -1503,7 +1503,9 @@ async function verifyAppRegistrationParameters(req, res) { appSpecification.version >= 8 && appSpecification.enterprise, ); - verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + if (isEnterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } // Decrypt enterprise specifications if needed const appSpecDecrypted = await checkAndDecryptAppSpecs( @@ -1581,7 +1583,9 @@ async function validateAppUpdate(appSpecification) { appSpecification.version >= 8 && appSpecification.enterprise, ); - verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + if (isEnterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } const decryptedSpecs = await checkAndDecryptAppSpecs(appSpecification, { daemonHeight }); diff --git a/ZelBack/src/services/utils/appSpecHelpers.js b/ZelBack/src/services/utils/appSpecHelpers.js index 8a891b2e6f..3177eee7a4 100644 --- a/ZelBack/src/services/utils/appSpecHelpers.js +++ b/ZelBack/src/services/utils/appSpecHelpers.js @@ -326,7 +326,9 @@ async function getAppFiatAndFluxPrice(req, res) { throw new Error('Daemon not yet synced.'); } const daemonHeight = syncStatus.data.height; - verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + if (appSpecification.version >= 8 && appSpecification.enterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } appSpecification = await checkAndDecryptAppSpecs(appSpecification, { daemonHeight }); const appSpecFormatted = specificationFormatter(appSpecification); diff --git a/tests/unit/appSpecHelpers.test.js b/tests/unit/appSpecHelpers.test.js index 9c700935e6..0078d16ae5 100644 --- a/tests/unit/appSpecHelpers.test.js +++ b/tests/unit/appSpecHelpers.test.js @@ -870,13 +870,14 @@ describe('appSpecHelpers tests', () => { }); [ - ['an invalid app name', { name: 'Price_Test' }], - ['an invalid owner', { owner: 'my-owner' }], + ['an app name the rule would reject', { name: 'Price_Test' }], + ['an owner the rule would reject', { owner: 'marketplace' }], ].forEach(([description, change]) => { - it(`should refuse a spec with ${description}`, async () => { + // A non-enterprise spec never reaches the decrypt, so its name and owner + // are not checked here: it is priced as before. + it(`should price a non-enterprise spec with ${description}`, async () => { const response = await quote(buildSpec('/data', change)); - expect(response.status).to.equal('error'); - expect(response.data.message).to.equal('Invalid app name or owner'); + expect(response.status).to.equal('success'); }); it(`should refuse an enterprise spec with ${description} before decrypting it`, async () => { @@ -885,6 +886,14 @@ describe('appSpecHelpers tests', () => { }); }); + it('should price a marketplace spec: version 8, an empty enterprise field, a placeholder owner', async () => { + // The shape the marketplace install dialog sends. + const response = await quote(buildSpec('/data', { + version: 8, enterprise: '', owner: 'marketplace', expire: 22000, contacts: [], geolocation: [], nodes: [], staticip: false, + })); + expect(response.status).to.equal('success'); + }); + it('should decrypt an enterprise spec with a valid app name and owner', async () => { // Off ArcaneOS the decrypt refuses, which shows the spec reached it. const response = await quote(buildSpec('/data', { version: 8, enterprise: 'blob' })); diff --git a/tests/unit/appValidator.test.js b/tests/unit/appValidator.test.js index 37eb1a2b18..5594820ab7 100644 --- a/tests/unit/appValidator.test.js +++ b/tests/unit/appValidator.test.js @@ -1092,9 +1092,8 @@ describe('appValidator spec verification route tests', () => { [ ['an invalid app name', { name: 'verify_test' }], ['an invalid owner', { owner: 'my-owner' }], - ['an invalid app name in a spec that is not enterprise', { version: 4, enterprise: undefined, name: 'verify_test' }], ].forEach(([description, change]) => { - it(`${route} should refuse a spec with ${description}`, async () => { + it(`${route} should refuse an enterprise spec with ${description}`, async () => { const response = await post(handler, { ...spec, ...change }); expect(response.status).to.equal('error'); expect(response.data.message).to.equal('Invalid app name or owner'); diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 53e1dd9e7a..650b9fa136 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -2106,9 +2106,8 @@ describe('registryManager registerAppGlobalyApi tests', () => { [ ['an invalid app name', { name: 'register_test' }], ['an invalid owner', { owner: 'my-owner' }], - ['an invalid app name in a spec that is not enterprise', { version: 4, enterprise: undefined, name: 'register_test' }], ].forEach(([description, change]) => { - it(`should refuse a spec with ${description}`, async () => { + it(`should refuse an enterprise spec with ${description}`, async () => { const response = await register({ ...spec, ...change }); expect(response.status).to.equal('error'); expect(response.data.message).to.equal('Invalid app name or owner'); diff --git a/tests/unit/updateAppGlobaly.test.js b/tests/unit/updateAppGlobaly.test.js index 516f680ece..baa187f639 100644 --- a/tests/unit/updateAppGlobaly.test.js +++ b/tests/unit/updateAppGlobaly.test.js @@ -109,15 +109,20 @@ describe('updateAppGlobaly tests', () => { }); [ - ['an invalid app name', { name: 'Submit_App' }], + ['an invalid app name', { name: 'Submit-App!' }], ['an invalid owner', { owner: 'my-owner' }], - ['an invalid owner in an enterprise spec', { version: 8, enterprise: 'blob', owner: 'my-owner' }], ].forEach(([description, change]) => { - it(`should refuse an update with ${description} before reading the spec`, async () => { + it(`should refuse an enterprise update with ${description} before reading the spec`, async () => { sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 3003100 } }); await advancedWorkflows.updateAppGlobaly({ - appSpecification: { ...spec, ...change }, timestamp: Date.now(), signature: 'sig', type: 'fluxappupdate', version: 1, + appSpecification: { + ...spec, version: 8, enterprise: 'blob', ...change, + }, + timestamp: Date.now(), + signature: 'sig', + type: 'fluxappupdate', + version: 1, }).then( () => expect.fail('should have refused'), (error) => expect(error.message).to.equal('Invalid app name or owner'), From f220a9111ff8c0c149085b41caee3a3a9cedb560 Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 08:37:05 +0000 Subject: [PATCH 367/369] refactor(apps): remove dead enterprise-owner lookups encryptEnterpriseWithAes had no callers. checkAndDecryptAppSpecs computed an owner it never used; the decrypt reads the spec's own owner. Co-Authored-By: Claude Opus 4.8 --- .../src/services/utils/enterpriseHelper.js | 88 ------------------- 1 file changed, 88 deletions(-) diff --git a/ZelBack/src/services/utils/enterpriseHelper.js b/ZelBack/src/services/utils/enterpriseHelper.js index 2b556ff760..c29ce9803e 100644 --- a/ZelBack/src/services/utils/enterpriseHelper.js +++ b/ZelBack/src/services/utils/enterpriseHelper.js @@ -158,7 +158,6 @@ async function checkAndDecryptAppSpecs(appSpec, options = {}) { const appSpecs = JSON.parse(JSON.stringify(appSpec)); let daemonHeight = options.daemonHeight || null; - let appOwner = options.owner || null; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -170,21 +169,6 @@ async function checkAndDecryptAppSpecs(appSpec, options = {}) { }; let appsQuery = null; - if (!appOwner) { - log.info(`Searching register permanent messages for ${appSpecs.name} to get registration message`); - appsQuery = { - 'appSpecifications.name': appSpecs.name, - type: 'fluxappregister', - }; - const permanentAppMessage = await dbHelper.findInDatabase(database, globalAppsMessages, appsQuery, projection); - if (permanentAppMessage.length > 0) { - const lastAppRegistration = permanentAppMessage[permanentAppMessage.length - 1]; - appOwner = lastAppRegistration.owner; - } else { - appOwner = appSpec.owner; - } - } - if (!daemonHeight) { log.info(`Searching register permanent messages for ${appSpecs.name} to get latest update`); appsQuery = { @@ -281,80 +265,8 @@ async function encryptEnterpriseFromSession(appSpec, daemonHeight, enterpriseKey throw new Error('Error encrypting enterprise object.'); } -/** - * Encrypts enterprise content with AES - * @param {object} enterprise - Content to be encrypted - * @param {string} appName - Application name - * @param {number} daemonHeight - Daemon block height (optional) - * @param {string} owner - Application owner (optional) - * @returns {Promise} Encrypted enterprise content - */ -async function encryptEnterpriseWithAes(enterprise, appName, daemonHeight = null, owner = null) { - let block = daemonHeight; - let appOwner = owner; - - if (!isArcane) { - throw new Error('Application Specifications can only be validated on a node running Arcane OS.'); - } - - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - const globalAppsMessages = config.database.appsglobal.collections.appsMessages; - const projection = { - projection: { - _id: 0, - }, - }; - let appsQuery = null; - - if (!appOwner) { - log.info(`Searching register permanent messages for ${appName} to get registration message`); - appsQuery = { - 'appSpecifications.name': appName, - type: 'fluxappregister', - }; - const permanentAppMessage = await dbHelper.findInDatabase(database, globalAppsMessages, appsQuery, projection); - const lastAppRegistration = permanentAppMessage[permanentAppMessage.length - 1]; - appOwner = lastAppRegistration.owner; - } - - if (!block) { - log.info(`Searching register permanent messages for ${appName} to get latest update`); - appsQuery = { - 'appSpecifications.name': appName, - }; - const allPermanentAppMessage = await dbHelper.findInDatabase(database, globalAppsMessages, appsQuery, projection); - const lastUpdate = allPermanentAppMessage[allPermanentAppMessage.length - 1]; - block = lastUpdate.height; - } - - const jsonEnterprise = JSON.stringify(enterprise); - const base64JsonEnterprise = Buffer.from(jsonEnterprise).toString('base64'); - - const inputData = JSON.stringify({ - fluxID: appOwner, - appName, - message: base64JsonEnterprise, - blockHeight: block, - }); - - const dataReturned = await benchmarkService.encryptMessage(inputData); - const { status, data } = dataReturned; - if (status === 'success') { - const dataParsed = JSON.parse(data); - const newEnterprise = status === 'success' && dataParsed.status === 'ok' ? dataParsed.message : null; - if (newEnterprise) { - return newEnterprise; - } - throw new Error('Error encrypting application specifications.'); - } else { - throw new Error('Error getting encrypted specifications.'); - } -} - module.exports = { checkAndDecryptAppSpecs, - encryptEnterpriseWithAes, encryptEnterpriseFromSession, decryptEnterpriseFromSession, decryptAesKeyWithRsaKey, From 04ddc7e2ed289b950bb3dfc8f842173cadce1455 Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 09:19:58 +0000 Subject: [PATCH 368/369] fix(enterprise): decrypt exactly the specs the route checks treat as enterprise checkAndDecryptAppSpecs now decrypts only when version >= 8 and an enterprise blob is present, the rule the route checks apply. A spec with a missing or non-numeric version is returned unchanged instead of being decrypted. Co-Authored-By: Claude Opus 5.5 --- .../src/services/utils/enterpriseHelper.js | 2 +- tests/unit/enterpriseHelper.test.js | 52 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 tests/unit/enterpriseHelper.test.js diff --git a/ZelBack/src/services/utils/enterpriseHelper.js b/ZelBack/src/services/utils/enterpriseHelper.js index c29ce9803e..9831d76d2d 100644 --- a/ZelBack/src/services/utils/enterpriseHelper.js +++ b/ZelBack/src/services/utils/enterpriseHelper.js @@ -146,7 +146,7 @@ async function decryptEnterpriseFromSession(base64Encrypted, appName, daemonHeig * @returns {Promise} Decrypted specifications */ async function checkAndDecryptAppSpecs(appSpec, options = {}) { - if (!appSpec || appSpec.version < 8 || !appSpec.enterprise) { + if (!appSpec || !(appSpec.version >= 8 && appSpec.enterprise)) { return appSpec; } diff --git a/tests/unit/enterpriseHelper.test.js b/tests/unit/enterpriseHelper.test.js new file mode 100644 index 0000000000..077fe83a39 --- /dev/null +++ b/tests/unit/enterpriseHelper.test.js @@ -0,0 +1,52 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +const MODULE_PATH = '../../ZelBack/src/services/utils/enterpriseHelper'; + +// isArcane is read when the module loads, so FLUXOS_PATH is set around the load. Reaching the +// database is the first step of a decrypt; the stub throws so a test sees whether it was reached. +function load() { + const previous = process.env.FLUXOS_PATH; + process.env.FLUXOS_PATH = '/tmp/fluxos'; + const dbHelper = { databaseConnection: sinon.stub().throws(new Error('decrypt reached')) }; + try { + const m = proxyquire(MODULE_PATH, { + '../dbHelper': dbHelper, + '../benchmarkService': {}, + '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, + }); + return { module: m, dbHelper }; + } finally { + if (previous === undefined) delete process.env.FLUXOS_PATH; + else process.env.FLUXOS_PATH = previous; + } +} + +describe('enterpriseHelper', () => { + afterEach(() => sinon.restore()); + + describe('checkAndDecryptAppSpecs', () => { + // A spec is enterprise exactly when version >= 8 and it carries an enterprise blob; the + // route checks use the same rule, so every spec that reaches a decrypt has passed them. + it('decrypts a version 8 spec with an enterprise blob', async () => { + const { module: m, dbHelper } = load(); + await m.checkAndDecryptAppSpecs({ version: 8, name: 'app', enterprise: 'blob' }) + .then(() => expect.fail('resolved without a decrypt'), (error) => expect(error.message).to.equal('decrypt reached')); + sinon.assert.calledOnce(dbHelper.databaseConnection); + }); + + [ + ['no version', { name: 'app', enterprise: 'blob' }], + ['a non-numeric version', { version: 'eight', name: 'app', enterprise: 'blob' }], + ['version 7', { version: 7, name: 'app', enterprise: 'blob' }], + ['an empty enterprise', { version: 8, name: 'app', enterprise: '' }], + ].forEach(([label, spec]) => { + it(`returns a spec with ${label} unchanged, without a decrypt`, async () => { + const { module: m, dbHelper } = load(); + expect(await m.checkAndDecryptAppSpecs(spec)).to.equal(spec); + sinon.assert.notCalled(dbHelper.databaseConnection); + }); + }); + }); +}); From c10a308a9a48011d9a3accb8e35a7d2088df28ee Mon Sep 17 00:00:00 2001 From: David White Date: Sun, 25 Oct 2026 14:30:00 +0000 Subject: [PATCH 369/369] chore(release): carry the 8.22.0 release fingerprint Step 2 of RELEASING.md for the development -> master release PR. helpers/hashes.json gains b9bfb6578c581b007f92137826c01cb1, the md5 over ./ZelBack at this tip. The file is outside ZelBack, so committing it does not move the fingerprint. Co-Authored-By: Claude Opus 5.5 --- helpers/hashes.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/helpers/hashes.json b/helpers/hashes.json index a53d06e051..e2995661bb 100644 --- a/helpers/hashes.json +++ b/helpers/hashes.json @@ -14,5 +14,6 @@ "16379f485d76083906f1fd19790dd1c4", "adcaa6996078beaff5d62fe95b2774d2", "b76e4ee20664bc6db7a91276f98f66b7", - "0840c013921a322079de7d694417ff42" + "0840c013921a322079de7d694417ff42", + "b9bfb6578c581b007f92137826c01cb1" ]