diff --git a/ZelBack/config/default.js b/ZelBack/config/default.js index f7b06e863e..549f7fa7f6 100644 --- a/ZelBack/config/default.js +++ b/ZelBack/config/default.js @@ -40,9 +40,32 @@ module.exports = { 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIfVQ1HBlOYD61OEzV19gSWPG6yVWvz+TyHXflXIHsPg fluxadm-legacy', ], }, + // A thread of a process FluxOS depends on that stays in the kernel without + // running for one interval holds the node back from new placements, and for + // stuckIntervals intervals takes it out of service until it reboots. Its apps + // are then asked to stop all at once, and whatever of them can still run after + // stopWindowMs is killed: longer than the slowest removal of a component seen on + // the fleet (39 s), which stops it first. + hostStall: { + readIntervalMs: 5 * 60 * 1000, + stuckIntervals: 6, + stopWindowMs: 60 * 1000, + }, database: { url: '127.0.0.1', port: 27017, + // How long one round trip, or the wait for a pooled connection, may take + // before the call fails. callTimeoutMs bounds every call on the main client: + // over four times the slowest healthy round trip seen on the fleet. + // maintenanceCallTimeoutMs bounds the client for the named long operations: + // index builds, the explorer restore's deletes and the registry rebuild. + callTimeoutMs: 60 * 1000, + maintenanceCallTimeoutMs: 15 * 60 * 1000, + // The database is suspect, and the node takes on no new apps, once + // suspectRoundTrips round trips have run out within suspectWindowMs. It stops + // being suspect suspectWindowMs after the last one. + suspectRoundTrips: 4, + suspectWindowMs: 30 * 60 * 1000, local: { database: `${dbPrefix}zelfluxlocal`, collections: { @@ -79,6 +102,7 @@ module.exports = { collections: { appsInformation: 'zelappsinformation', appsRuntimeState: 'zelappsruntimestate', // node-local per-component controller state: desiredState, restartHistory (crash backoff), last exit + appsPlacements: 'zelappsplacements', // when this node took each app it holds, kept through redeploys }, }, appsglobal: { @@ -107,6 +131,10 @@ module.exports = { upnp: { gatewayUrl: '', nodeIp: '', + // a removal of this node's own mapping the router refuses is asked again + // after this long, the wait doubling up to removalRetryMaxMs + removalRetryFirstMs: 30 * 1000, + removalRetryMaxMs: 60 * 1000, }, benchmark: { host: '127.0.0.1', @@ -136,6 +164,11 @@ module.exports = { // Docker's networking rules and its patch releases fixed them; 28.1.1 is the // oldest release the fleet runs in numbers. minimumDockerAllowedVersion: '28.1.1', + // The MongoDB a node runs: the oldest release FluxOS supports. From 6.0 a + // $group or $sort past the memory limit spills to disk rather than failing, + // which the registry rebuild and the duplicate merges at boot rely on; the + // patch level is the oldest 6.0 release the fleet runs in numbers. + minimumMongoDbAllowedVersion: '6.0.27', fluxTeamFluxID: '1hjy4bCYBJr4mny4zCE85J94RXa8W6q37', // A list, so support can be granted to (or revoked from) an identity without // touching every privilege check. A bare string is still read as a one entry @@ -448,9 +481,10 @@ module.exports = { peerShortfallDosPerMinute: 0.13, // While this node knows no address of its own it asks benchmark to detect its // public IP this often, and is out of service after this many asks find none. - // Benchmark caches its answer for 69 s. - publicIpAskIntervalMs: 90 * 1000, - publicIpAsksBeforeDos: 8, + // Benchmark caches an address it found for 69 s and none it failed to find, so + // each ask while none is known is a fresh lookup against public IP services. + publicIpAskIntervalMs: 15 * 1000, + publicIpAsksBeforeDos: 48, // The apps port check. timeoutsMs: how long it waits before testing again // after a pass, an error, a failed port, once its count reaches the DOS // threshold, and after an app error. thresholds: once portsHighEdge ports @@ -562,8 +596,6 @@ module.exports = { daemonPONFork: 2020000, // block height where PON (Proof of Node) fork activates - chain works 4x faster after this block blocksAllowanceInterval: 1000, // ap differences can be in 1000s - more than 1 day removeBlocksAllowanceIntervalBlock: 1625000, // after this block we can start having app updates without extending subscription - block expected in April 19th 2024 - ownerAppAllowance: 1000, // a by-name local install (fluxteam only) runs for this amount of blocks before the expiry sweep removes it - temporaryAppAllowance: 200, // in case of any user installing some temporary app message for testing purposes, the app will run for this many blocks expireFluxAppsPeriod: 100, // every 100 blocks we run a check that deletes apps specifications and stops/removes the application from existence if it has been lastly updated more than 22k blocks ago updateFluxAppsPeriod: 9, // every 9 blocks we check for reinstalling of old application versions removeFluxAppsPeriod: 11, // every 11 blocks we check for more than maximum number of instances of an application @@ -577,6 +609,9 @@ module.exports = { bootDelayMultiplier: 1, spawnDelayMs: 0, removalSpacingMs: 60000, + // How long one app's removal may take, when every app is removed, before the + // next app's starts. A removal past it keeps running. + appRemovalDeadlineMs: 5 * 60 * 1000, // Per-document expiry for the ephemeral app collections, in seconds, read by // appConstants.js. Each record carries its own deadline and the collection // index is expireAt/expireAfterSeconds:0, so changing one of these takes @@ -645,6 +680,8 @@ module.exports = { connectionBackoffMs: [120000, 300000, 600000, 900000], nodeMonitorIntervalMs: 1200000, nodeMonitorRemovalDelayMs: 60000, + // events per message of an answer to a peer's state sync + syncAnswerBatchSize: 2000, // Residential-node staging. The placement hold is immediate and is not // tunable; these pace only the part that moves customer data. residentialCheckIntervalMs: 6 * 60 * 60 * 1000, // re-evaluate the verdict diff --git a/ZelBack/src/services/appDatabase/registryManager.js b/ZelBack/src/services/appDatabase/registryManager.js index b1ef43256a..268acf896a 100644 --- a/ZelBack/src/services/appDatabase/registryManager.js +++ b/ZelBack/src/services/appDatabase/registryManager.js @@ -10,6 +10,7 @@ const globalState = require('../utils/globalState'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../utils/appSyncEvents'); // Removed appsService to avoid circular dependency - will use dynamic require where needed const { checkAndDecryptAppSpecs, encryptEnterpriseFromSession } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const { specificationFormatter, updateToLatestAppSpecifications } = require('../utils/appUtilities'); const placementFeasibility = require('../appPlacement/placementFeasibility'); const { withRegistryWrite } = require('./registryWriteLock'); @@ -242,18 +243,32 @@ async function addressMovesTo(address) { return moves; } +/** + * The query condition for every stored address at an IP, whatever its port. + * + * Every location and every move is stored under its node's address in ip:port + * form, so the IP and the colon after it match exactly the addresses at that + * IP: never one that only begins with it. + * @param {string} ip An IP, or a socket address whose IP is meant. + * @returns {RegExp|null} null when there is no IP + */ +function storedAtIp(ip) { + const bare = extractIp(ip); + if (!bare) return null; + return new RegExp(`^${bare.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}:`); +} + /** * The socket addresses at `ip` that a recorded move leaves. * @param {string} ip An IP, or a socket address whose IP is meant. * @returns {Promise} */ async function addressesMovedFromIp(ip) { - const bare = extractIp(ip); - if (!bare) return []; + const atIp = storedAtIp(ip); + if (!atIp) return []; const collection = dbHelper.databaseConnection().db(config.database.appsglobal.database).collection(globalAppStateEvents); - const escaped = bare.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); const events = await collection.find( - { ip: new RegExp(`^${escaped}:`), type: 'ipchanged', expireAt: { $gt: new Date() } }, + { ip: atIp, type: 'ipchanged', expireAt: { $gt: new Date() } }, { projection: { _id: 0, ip: 1 } }, ).toArray(); return [...new Set(events.map((event) => event.ip))]; @@ -1611,90 +1626,29 @@ async function checkApplicationRegistrationNameConflicts(appSpecFormatted, hash) return true; } -/** - * Get app specification from database - * @param {string} appName - Application name - * @returns {Promise} App specification - */ -async function getAppSpecificationFromDb(appName) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - - const query = { name: new RegExp(`^${appName}$`, 'i') }; - const projection = { projection: { _id: 0 } }; - - const appSpec = await dbHelper.findOneInDatabase(database, globalAppsInformation, query, projection); - return appSpec; - } catch (error) { - log.error(`Error getting app specification from database: ${error.message}`); - return null; - } -} - -/** - * Get all apps information (both global and local) - * @returns {Promise} Array of all app information - */ -async function getAllAppsInformation() { - try { - const allApps = await availableApps(); - return allApps; - } catch (error) { - log.error(`Error getting all apps information: ${error.message}`); - return []; - } -} - /** * Get installed apps information - * @returns {Promise} Array of installed apps + * @returns {Promise} Array of installed apps. Rejects when the read fails: + * an unread list is not an empty one. */ async function getInstalledApps() { - try { - const localDb = dbHelper.databaseConnection(); - const localDatabase = localDb.db(config.database.appslocal.database); - - const query = {}; - const projection = { projection: { _id: 0 } }; - - const installedApps = await dbHelper.findInDatabase(localDatabase, localAppsInformation, query, projection); - return installedApps; - } catch (error) { - log.error(`Error getting installed apps: ${error.message}`); - return []; - } -} - -/** - * Get running apps information - * @returns {Promise} Array of running apps - */ -async function getRunningApps() { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - - const query = {}; - const projection = { projection: { _id: 0 } }; - - const runningApps = await dbHelper.findInDatabase(database, globalAppsLocations, query, projection); - return runningApps; - } catch (error) { - log.error(`Error getting running apps: ${error.message}`); - return []; - } + const localDb = dbHelper.databaseConnection(); + const localDatabase = localDb.db(config.database.appslocal.database); + const projection = { projection: { _id: 0 } }; + return dbHelper.findInDatabase(localDatabase, localAppsInformation, {}, projection); } /** * To get all apps running on a specific IP address. Returns all apps running on this ip - * @param {string} ip IP address to check + * @param {string} ip An IP, or a socket address whose IP is meant. * @returns {Promise} Array of apps running on the specified IP */ async function getRunningAppIpList(ip) { + const atIp = storedAtIp(ip); + if (!atIp) return []; const dbopen = dbHelper.databaseConnection(); const database = dbopen.db(config.database.appsglobal.database); - const query = { ip: new RegExp(`^${ip}`) }; + const query = { ip: atIp }; const projection = { projection: { _id: 0, @@ -1734,31 +1688,6 @@ function registrationInformation(_req, res) { } } -/** - * Get all global applications from database - * @param {string[]} proj - Optional projection fields - * @returns {Promise} Array of global applications - */ -async function getAllGlobalApplications(proj = []) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - const query = {}; - const wantedProjection = { - _id: 0, - }; - proj.forEach((field) => { - wantedProjection[field] = 1; - }); - const projection = { projection: wantedProjection, sort: { height: 1 } }; // ensure sort from oldest to newest - const results = await dbHelper.findInDatabase(database, globalAppsInformation, query, projection); - return results; - } catch (error) { - log.error(error); - return []; - } -} - /** * Remove expired applications from global database and local installations * @returns {Promise} Completion status @@ -1830,6 +1759,9 @@ async function expireGlobalApplications() { const appsToRemove = []; const locallyExpired = []; appsInstalled.forEach((app) => { + // An app being tested has no height until it is registered, and its test + // removes it when it ends. + if (globalState.testInstallingApps.has(app.name)) return; if (appNamesToExpire.includes(app.name)) { appsToRemove.push(app); } else if (!app.height) { @@ -1922,28 +1854,26 @@ async function isGoverningAppMessage(appName, hash) { * force (appMessageChain): the one that governs the name in the message log, and alive at the * current block. Any other message changes nothing: a newer one governs the app, or its term has ended. * A registration and an update are stored alike, so an update renews an app this node's expiry - * pass has already removed, and this node holds what a rebuild from the log would. + * pass has already removed, and this node holds what a rebuild from the log would. A row that + * already holds this message at this height is left as it is. * @param {object} appSpecs the message's specifications, with its hash and height - * @returns {Promise} false when the write failed + * @returns {Promise} Rejects when a read or the write fails: the caller decides what an + * unwritten registry row costs it. */ async function storeAppSpecificationInForce(appSpecs) { return withRegistryWrite(async () => { - try { - if (!await isGoverningAppMessage(appSpecs.name, appSpecs.hash)) return true; - const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); - const currentHeight = syncStatus && syncStatus.data ? syncStatus.data.height : 0; - if (!appMessageChain.isInForce(appSpecs.height, appSpecs.expire, currentHeight)) return true; - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await dbHelper.replaceOneInDatabase(database, globalAppsInformation, { name: appSpecs.name }, appSpecs, { upsert: true }); - fluxEventBus.publish('app:specStored', { name: appSpecs.name, hash: appSpecs.hash }); - await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsLocations, { name: appSpecs.name }); - await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsBroadcasts, { 'data.name': appSpecs.name }); - return true; - } catch (error) { - log.error(`storeAppSpecificationInForce failed for ${appSpecs.name}: ${error.message}`); - return false; - } + if (!await isGoverningAppMessage(appSpecs.name, appSpecs.hash)) return; + const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); + const currentHeight = syncStatus && syncStatus.data ? syncStatus.data.height : 0; + if (!appMessageChain.isInForce(appSpecs.height, appSpecs.expire, currentHeight)) return; + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + const held = await dbHelper.findOneInDatabase(database, globalAppsInformation, { name: appSpecs.name }, { projection: { _id: 0, hash: 1, height: 1 } }); + if (held && held.hash === appSpecs.hash && held.height === appSpecs.height) return; + await dbHelper.replaceOneInDatabase(database, globalAppsInformation, { name: appSpecs.name }, appSpecs, { upsert: true }); + fluxEventBus.publish('app:specStored', { name: appSpecs.name, hash: appSpecs.hash }); + await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsLocations, { name: appSpecs.name }); + await dbHelper.removeDocumentsFromCollection(database, globalAppsInstallingErrorsBroadcasts, { 'data.name': appSpecs.name }); }); } @@ -2030,8 +1960,12 @@ async function reindexGlobalAppsInformation() { return rebuildFollowing; } +// Hash records corrected per round trip. +const HASH_RECONSTRUCTION_BATCH = 5000; + /** - * Reconstruct app messages hash collection by validating hash records against actual messages + * Reconstruct app messages hash collection by validating hash records against actual messages. + * Reads only the fields it compares, and writes its corrections HASH_RECONSTRUCTION_BATCH at a time. * @returns {Promise} Success message */ async function reconstructAppMessagesHashCollection() { @@ -2041,11 +1975,13 @@ async function reconstructAppMessagesHashCollection() { const databaseDaemon = db.db(config.database.daemon.database); const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); const currentHeight = syncStatus.data.height || 0; - const query = {}; - const projection = { projection: { _id: 0 } }; - const permanentMessages = await dbHelper.findInDatabase(databaseApps, globalAppsMessages, query, projection); - const appHashes = await dbHelper.findInDatabase(databaseDaemon, appsHashesCollection, query, projection); + const permanentMessages = await dbHelper.findInDatabase(databaseApps, globalAppsMessages, {}, { projection: { _id: 0, hash: 1 } }); + const appHashes = await dbHelper.findInDatabase(databaseDaemon, appsHashesCollection, {}, { + projection: { + _id: 0, hash: 1, txid: 1, height: 1, message: 1, messageNotFound: 1, retryFromHeight: 1, + }, + }); const permHashSet = new Set(permanentMessages.map((m) => m.hash)); const ops = []; @@ -2083,8 +2019,9 @@ async function reconstructAppMessagesHashCollection() { } let changed = 0; - if (ops.length > 0) { - const result = await databaseDaemon.collection(appsHashesCollection).bulkWrite(ops, { ordered: false }); + for (let i = 0; i < ops.length; i += HASH_RECONSTRUCTION_BATCH) { + // eslint-disable-next-line no-await-in-loop + const result = await databaseDaemon.collection(appsHashesCollection).bulkWrite(ops.slice(i, i + HASH_RECONSTRUCTION_BATCH), { ordered: false }); changed += result.modifiedCount; } @@ -2212,6 +2149,10 @@ async function registerAppGlobalyApi(req, res) { } const daemonHeight = syncStatus.data.height; + if (appSpecification.version >= 8 && appSpecification.enterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } + const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecification, { @@ -2617,13 +2558,9 @@ module.exports = { storeAppSpecificationInForce, governingAppMessage, isGoverningAppMessage, - getAppSpecificationFromDb, - getAllAppsInformation, getInstalledApps, - getRunningApps, getRunningAppIpList, registrationInformation, - getAllGlobalApplications, expireGlobalApplications, reindexGlobalAppsInformation, reindexGlobalAppsLocation, diff --git a/ZelBack/src/services/appLifecycle/advancedWorkflows.js b/ZelBack/src/services/appLifecycle/advancedWorkflows.js index df5d344c68..54576b2a37 100644 --- a/ZelBack/src/services/appLifecycle/advancedWorkflows.js +++ b/ZelBack/src/services/appLifecycle/advancedWorkflows.js @@ -20,6 +20,7 @@ const { collateralOutpoint, nodesNameThisNode } = require('../utils/nodePinning' const fluxEventBus = require('../utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../utils/addressMoveEvents'); const { InstallOutcome } = require('../utils/installOutcome'); +const appRebuild = require('./appRebuild'); const generalService = require('../generalService'); const placementFeasibility = require('../appPlacement/placementFeasibility'); // eslint-disable-next-line no-unused-vars @@ -36,6 +37,7 @@ const { const { specificationFormatter } = require('../utils/appSpecHelpers'); const { compareInstanceSeniority } = require('../utils/instanceOrdering'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const volumeService = require('../utils/volumeService'); const mountParser = require('../utils/mountParser'); const appReconciler = require('../appMonitoring/appReconciler'); @@ -54,6 +56,7 @@ const appsRuntimeState = require('../appManagement/appsRuntimeState'); const { stopAppMonitoring } = require('../appManagement/appInspector'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const globalState = require('../utils/globalState'); +const folderStates = require('../appMonitoring/folderStates'); const appNetworkLinker = require('./appNetworkLinker'); const { Privilege, authOf } = require('../utils/privileges'); const { AsyncLock } = require('../utils/asyncLock'); @@ -295,24 +298,23 @@ function hasV8ComponentStructureChange(appSpecifications, installedApp) { return componentCountChanged || componentNamesChanged; } -// Get strict application specifications +/** + * The registry row of `appName`, or null when it has none. Rejects when the + * registry cannot be read. + * @param {string} appName + * @returns {Promise} + */ async function getStrictApplicationSpecifications(appName) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); - const query = { name: appName }; - const projection = { - projection: { - _id: 0, - }, - }; - const appInfo = await dbHelper.findOneInDatabase(database, globalAppsInformation, query, projection); - return appInfo; - } catch (error) { - log.error(`Error getting strict app specifications for ${appName}:`, error); - return null; - } + const query = { name: appName }; + const projection = { + projection: { + _id: 0, + }, + }; + return dbHelper.findOneInDatabase(database, globalAppsInformation, query, projection); } /** @@ -468,12 +470,17 @@ async function findAndRestoreNonEnterpriseSpecs(installedApp) { return specsForRemoval; } +// How long after an enterprise check a database read stopped it runs again. +const ENTERPRISE_CHECK_RETRY_MS = 5 * 60 * 1000; + /** * Check and remove enterprise apps (v8+) running on non-arcaneOS nodes. - * This function runs once at startup to clean up incompatible apps. + * This function runs once at startup to clean up incompatible apps, and again + * after ENTERPRISE_CHECK_RETRY_MS while a database read stops it. * @returns {Promise} Completion status */ async function checkAndRemoveEnterpriseAppsOnNonArcane() { + let unread = false; try { // Skip if running on arcaneOS if (isArcane) { @@ -487,6 +494,7 @@ async function checkAndRemoveEnterpriseAppsOnNonArcane() { const installedAppsRes = await getInstalledAppsFromDb(); if (installedAppsRes.status !== 'success') { log.error('Failed to get installed apps for enterprise check'); + unread = true; return; } @@ -538,6 +546,7 @@ async function checkAndRemoveEnterpriseAppsOnNonArcane() { } } catch (error) { log.error(`Error processing app ${installedApp.name} for enterprise check:`, error); + if (dbHelper.isDatabaseError(error)) unread = true; // Continue with next app even if this one fails } } @@ -545,6 +554,13 @@ async function checkAndRemoveEnterpriseAppsOnNonArcane() { log.info('Enterprise app compatibility check completed'); } catch (error) { log.error('Error in checkAndRemoveEnterpriseAppsOnNonArcane:', error); + if (dbHelper.isDatabaseError(error)) unread = true; + } finally { + if (unread) { + log.warn(`Enterprise app compatibility check could not read every app, running again in ${ENTERPRISE_CHECK_RETRY_MS / 60000} min`); + const timer = setTimeout(checkAndRemoveEnterpriseAppsOnNonArcane, ENTERPRISE_CHECK_RETRY_MS); + if (timer.unref) timer.unref(); + } } } @@ -664,7 +680,7 @@ async function createAppVolume(appSpecifications, appName, isComponent, res) { // Same dead incarnation, and this claim is published: peers rank a seed on what // each node says it holds, so the old volume's figures describe the one being // replaced and can win the election for a volume that is about to be empty. - globalState.folderHoldings?.delete(appId); + folderStates.recreated(appId); globalState.seedMarks.delete(appId); await execAsRoot('fallocate', ['-l', `${appSpecifications.hdd}G`, volumeFile]); const allocateSpace2 = { @@ -1021,6 +1037,17 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { } return InstallOutcome.BUSY; } + // A pending removal of the app removes whatever is installed in its place, so + // the app takes no install until that removal has finished. + if (globalState.departingApps.has(appSpecs.name)) { + const rStatus = messageHelper.createErrorMessage(`Flux App ${appSpecs.name} is being removed`); + log.error(rStatus); + if (res) { + res.write(serviceHelper.ensureString(rStatus)); + if (res.flush) res.flush(); + } + return InstallOutcome.BUSY; + } globalState.installationInProgress = true; acquired = true; const tier = await generalService.nodeTier().catch((error) => log.error(error)); @@ -1128,6 +1155,7 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name} in soft registration. Database insert returned undefined - likely duplicate key error or database failure. Aborting soft registration to prevent orphaned Docker containers.`); } log.info(`Database entry created for ${appSpecifications.name} BEFORE Docker container creation (soft registration)`); + folderStates.installed(appSpecifications); const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; const cpuTier = `cpu${tier}`; @@ -1224,6 +1252,12 @@ async function softRegisterAppLocally(appSpecs, componentSpecs, res) { res.write(serviceHelper.ensureString(errorResponse)); if (res.flush) res.flush(); } + if (dbHelper.isDatabaseError(error)) { + // A soft registration keeps the app's data, and the teardown below would not: + // a database that did not answer is no reason to destroy it. + log.warn(`Soft registration of ${appSpecs.name} stopped: the database did not answer (${error.message}). Nothing is torn down.`); + return InstallOutcome.DEFERRED; + } const removeStatus = messageHelper.createErrorMessage(`Error occured. Initiating Flux App ${appSpecs.name} removal`); log.info(removeStatus); log.warn(`REMOVAL REASON: Soft registration failure - ${appSpecs.name} failed during soft registration: ${error.message} (softRegisterAppLocally)`); @@ -1442,6 +1476,12 @@ async function mayTearDownToRebuild(appSpecs, res) { } }; + // A rebuilt app starts, and boot has yet to decide whether this one may. + if (globalState.unconfirmedApps.has(appSpecs.name)) { + report(`${appSpecs.name} is held until this node confirms it still holds it, redeploy deferred`); + return false; + } + const pinned = Array.isArray(appSpecs.nodes) ? appSpecs.nodes : []; if (pinned.length) { const collateral = await generalService.obtainNodeCollateralInformation().catch(() => null); @@ -1605,10 +1645,12 @@ async function softRedeploy(appSpecs, res) { if (res.flush) res.flush(); } globalState.softRedeployInProgress = false; + fluxEventBus.publish('app:softRedeployed', { name: appSpecs.name, outcome }); return; } log.info('Application softly redeployed'); globalState.softRedeployInProgress = false; + fluxEventBus.publish('app:softRedeployed', { name: appSpecs.name, outcome }); } catch (error) { log.info('Error on softRedeploy'); log.error(error); @@ -1634,6 +1676,20 @@ async function softRedeploy(appSpecs, res) { } return; } + if (dbHelper.isDatabaseError(error)) { + // The app is down and its volume is intact. A database that did not answer + // says nothing about the app, so it is left as a declined reinstall leaves + // it, never uninstalled at the cost of its data. + const notReinstalled = messageHelper.createErrorMessage( + `Application ${appSpecs.name} was not reinstalled: the database did not answer (${error.message})`, + ); + log.warn(notReinstalled); + if (res) { + res.write(serviceHelper.ensureString(notReinstalled)); + if (res.flush) res.flush(); + } + return; + } log.warn(`REMOVAL REASON: Soft redeploy failure - ${appSpecs.name} failed during soft redeploy: ${error.message} (softRedeploy)`); // eslint-disable-next-line global-require const appUninstaller = require('./appUninstaller'); @@ -1649,8 +1705,6 @@ async function softRedeploy(appSpecs, res) { * @param {object} res - Response object */ async function hardRedeploy(appSpecs, res) { - // eslint-disable-next-line global-require - const appUninstaller = require('./appUninstaller'); try { // Every operation, including the periodic reinstall pass - which sets its // flag before it tears anything down and holds it across the wait, so the @@ -1670,19 +1724,7 @@ async function hardRedeploy(appSpecs, res) { globalState.hardRedeployInProgress = true; log.warn(`REMOVAL REASON: Hard redeploy initiated - ${appSpecs.name} being removed as part of hard redeploy process (hardRedeploy)`); - await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); - const appRedeployResponse = messageHelper.createSuccessMessage('Application removed. Awaiting installation...'); - log.info(appRedeployResponse); - if (res) { - res.write(serviceHelper.ensureString(appRedeployResponse)); - if (res.flush) res.flush(); - } - // verify requirements - // eslint-disable-next-line global-require - const appInstaller = require('./appInstaller'); - await appInstaller.checkAppRequirements(appSpecs); - // register - const outcome = await appInstaller.registerAppLocally(appSpecs, undefined, res, false, true); // can throw + const outcome = await appRebuild.rebuildApp(appSpecs, res); if (outcome !== InstallOutcome.INSTALLED) { const notReinstalled = messageHelper.createErrorMessage( `Application ${appSpecs.name} was not reinstalled (${outcome})`, @@ -1698,12 +1740,17 @@ async function hardRedeploy(appSpecs, res) { log.info('Application redeployed'); globalState.hardRedeployInProgress = false; } catch (error) { + // rebuildApp settles every failure of its own, so this is a failure before it ran log.error(error); - log.warn(`REMOVAL REASON: Hard redeploy failure - ${appSpecs.name} failed during hard redeploy: ${error.message} (hardRedeploy)`); globalState.hardRedeployInProgress = false; - // endResponse false: redeployAPI opened this response and closes it. - await appUninstaller.removeAppLocally(appSpecs.name, res, true, false, true); - log.info(`Cleanup completed for ${appSpecs.name} after hard redeploy failure`); + const notStarted = messageHelper.createErrorMessage( + `Hard redeploy of ${appSpecs.name} failed before the app was taken down: ${error.message}`, + ); + log.warn(notStarted); + if (res) { + res.write(serviceHelper.ensureString(notStarted)); + if (res.flush) res.flush(); + } } } @@ -1862,6 +1909,12 @@ async function softRedeployComponent(appName, componentName, res) { + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.'); throw error; } + if (dbHelper.isDatabaseError(error)) { + // Uninstalling the whole app takes every other component's data with it, + // and a database that did not answer says nothing about any of them. + log.warn(`Component ${fullComponentName} was not reinstalled: the database did not answer (${error.message}). The app is not uninstalled.`); + throw error; + } log.warn(`REMOVAL REASON: Soft redeploy failure - ${appName} being removed after component ${fullComponentName} failed during soft redeploy: ${error.message} (softRedeployComponent)`); // endResponse false: the endpoint opened this response and closes it. await appUninstaller.removeAppLocally(appName, res, true, false, true); @@ -2006,6 +2059,12 @@ async function hardRedeployComponent(appName, componentName, res) { + 'No forced uninstall - the app is not known to be down, and convergence is left to the reconciler.'); throw error; } + if (dbHelper.isDatabaseError(error)) { + // Uninstalling the whole app takes every other component's data with it, + // and a database that did not answer says nothing about any of them. + log.warn(`Component ${fullComponentName} was not reinstalled: the database did not answer (${error.message}). The app is not uninstalled.`); + throw error; + } log.warn(`REMOVAL REASON: Hard redeploy failure - ${appName} being removed after component ${fullComponentName} failed during hard redeploy: ${error.message} (hardRedeployComponent)`); // endResponse false: the endpoint opened this response and closes it. await appUninstaller.removeAppLocally(appName, res, true, false, true); @@ -2540,64 +2599,29 @@ async function appDockerStop(appname, onWait) { } /** - * Helper function to restart app docker containers - * Ensures mount paths exist before restarting (important after Syncthing cleanup) - * @param {string} appname - App name + * Restarts an app once an update has rebuilt it, through the reconciler: each + * component gets a restart request, and the reconciler bounces the ones it keeps + * running on this node and starts none it keeps stopped. A g: component on a + * standby stays down - the election decides where it runs - and so does one its + * owner stopped. A stopped component the reconciler does start counts as + * restarted, so it is not bounced again once it runs. + * @param {string} appname App name, or `_` for one component. * @returns {Promise} */ async function appDockerRestart(appname) { - // eslint-disable-next-line global-require try { - // eslint-disable-next-line global-require - const { startAppMonitoring } = require('../appManagement/appInspector'); - // eslint-disable-next-line global-require - const registryManager = require('../appDatabase/registryManager'); - const mainAppName = appname.split('_')[1] || appname; - const isComponent = appname.includes('_'); - if (isComponent) { - // For component apps, fetch full specifications to ensure mount paths exist - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - // Find the specific component - const componentName = appname.split('_')[0]; - const componentSpec = appSpecs.compose.find((comp) => comp.name === componentName); - if (componentSpec && componentSpec.containerData) { - // Ensure mount paths exist before restarting (handles Syncthing cleanup) - // eslint-disable-next-line no-use-before-define - await volumeService.ensureMountPathsExist(componentSpec, mainAppName, true, appSpecs); - } - await dockerService.appDockerRestart(appname); - startAppMonitoring(appname); - } else { - const appSpecs = await registryManager.getApplicationSpecifications(mainAppName); - if (!appSpecs) { - throw new Error('Application not found'); - } - if (appSpecs.version <= 3) { - // Ensure mount paths exist before restarting (handles Syncthing cleanup) - if (appSpecs.containerData) { - // eslint-disable-next-line no-use-before-define - await volumeService.ensureMountPathsExist(appSpecs, mainAppName, false, null); - } - await dockerService.appDockerRestart(appname); - startAppMonitoring(appname); - } else { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecs.compose) { - // Ensure mount paths exist before restarting (handles Syncthing cleanup) - // eslint-disable-next-line no-await-in-loop - if (appComponent.containerData) { - // eslint-disable-next-line no-await-in-loop, no-use-before-define - await volumeService.ensureMountPathsExist(appComponent, appSpecs.name, true, appSpecs); - } - // eslint-disable-next-line no-await-in-loop - await dockerService.appDockerRestart(`${appComponent.name}_${appSpecs.name}`); - startAppMonitoring(`${appComponent.name}_${appSpecs.name}`); - } - } + const appSpecs = await registryManagerModule.getApplicationSpecifications(mainAppName); + if (!appSpecs) { + throw new Error('Application not found'); + } + const identifiers = appname.includes('_') || appSpecs.version <= 3 + ? [appname] + : appSpecs.compose.map((appComponent) => `${appComponent.name}_${appSpecs.name}`); + // eslint-disable-next-line no-restricted-syntax + for (const identifier of identifiers) { + // eslint-disable-next-line no-await-in-loop + await appReconciler.applyIntent(identifier, () => appsRuntimeState.requestRestart(identifier)); } } catch (error) { log.error(error); @@ -2732,6 +2756,18 @@ async function startOperatorStarted(identifier, appId) { * @param {object} req - Request object * @param {object} res - Response object */ +/** + * Refuses a task that starts an app this node is removing, or has not yet confirmed + * it still holds. + * @param {string} appName + * @returns {void} + * @throws {Error} When the app is held. + */ +function refuseHeldApp(appName) { + if (globalState.departingApps.has(appName)) throw new Error(`Refused: ${appName} is being removed from this node`); + if (globalState.unconfirmedApps.has(appName)) throw new Error(`Refused: ${appName} is held until this node confirms it still holds it`); +} + async function appendBackupTask(req, res) { let appname; let backup; @@ -2756,6 +2792,8 @@ async function appendBackupTask(req, res) { if (hasTrueBackup === false) { throw new Error('No backup jobs...'); } + // a backup ends by starting the app + refuseHeldApp(appname); // The claim, before any awaited work: a second request for the same app // finds it taken here rather than passing an emptied check and racing to // the archive alongside the first. Last in this synchronous block, so a @@ -2992,6 +3030,8 @@ async function appendRestoreTask(req, res) { if (hasTrueRestore === false) { throw new Error('No restore jobs...'); } + // a restore ends by starting the app + refuseHeldApp(appname); // The claim, before any awaited work: a second request for the same app // finds it taken here rather than passing an emptied check and racing to // the clear alongside the first. Last in this synchronous block, so a @@ -3524,6 +3564,9 @@ async function updateAppGlobaly(params) { const daemonHeight = syncStatus.data.height; const appSpecObj = serviceHelper.ensureObject(appSpecification); + if (appSpecObj.version >= 8 && appSpecObj.enterprise) { + verifyAppNameAndOwner(appSpecObj.name, appSpecObj.owner); + } const appSpecDecrypted = await checkAndDecryptAppSpecs(appSpecObj, { daemonHeight }); const appSpecFormatted = specificationFormatter(appSpecDecrypted); @@ -3756,23 +3799,6 @@ function isElectedPrimaryHere(appName, localSocketAddr, now = Date.now()) { return namesAPrimary ? false : null; } -/** - * The identifier of an app's `g:` component, or null when it has none. - * - * Derived the same way masterSlaveApps derives it - the app name below v4, and - * `_` above it - because it names the same thing: the one - * component that runs on a single node at a time and writes to the volume. - * @param {object} installedApp Locally installed app record. - * @returns {string|null} - */ -function gComponentIdentifier(installedApp) { - if (installedApp.version <= 3) { - return mountParser.isGComponent(installedApp.containerData) ? installedApp.name : null; - } - const component = (installedApp.compose || []).find((c) => mountParser.isGComponent(c.containerData)); - return component ? `${component.name}_${installedApp.name}` : null; -} - /** * Whether this node should give up an app, and why. * @@ -3808,7 +3834,10 @@ async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, // by the shared ordering so every node names the same surplus const ordered = [...runningAppList].sort((a, b) => compareInstanceSeniority(b, a)); const index = ordered.findIndex((x) => socketAddressesMatch(x.ip, localSocketAddr)); - const writer = gComponentIdentifier(installedApp); + // The app's g: components: run on one node at a time, by its primary, and + // together they are what that node writes. + const writers = mountParser.gComponentIdentifiers(installedApp); + const writer = writers.join(', '); const detail = `running on ${runningAppList.length} instances (max: ${minInstances}) and this node is the newest`; // "THE NEWEST STANDS ASIDE" IS A STAND-IN FOR "THE LEAST VALUABLE COPY @@ -3824,9 +3853,8 @@ async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, // not down, and interrupting the one node serving it to tidy up the count // is a worse outcome than the count being wrong for another pass. if (index === 0) { - // eslint-disable-next-line no-await-in-loop - const runsWriter = Boolean(writer) && Boolean(deps.isComponentRunningLocally) - && await deps.isComponentRunningLocally(writer); + const runsWriter = Boolean(deps.isComponentRunningLocally) + && (await Promise.all(writers.map((each) => deps.isComponentRunningLocally(each)))).includes(true); if (!runsWriter) return { giveUp: true, reason: 'SURPLUS', detail }; // Carried out to the evacuation gate rather than returned past it, for the // reason the second-newest branch below is: a node that is also draining @@ -3849,12 +3877,11 @@ async function reasonToGiveUpApp(installedApp, runningAppList, localSocketAddr, // Silence, a timeout, a refusal, "not running": all mean this node does // nothing, and nothing is exactly today's behaviour. The rule can only ever // fail towards no trim, never towards two. - if (index === 1 && writer && deps.liveness) { - const appId = dockerService.getAppIdentifier(writer); + if (index === 1 && writers.length && deps.liveness) { // eslint-disable-next-line no-await-in-loop const newestState = await peerComponentState(ordered[0].ip, { - appId, - identifier: writer, + appIds: writers.map((each) => dockerService.getAppIdentifier(each)), + identifier: writers[0], appName: installedApp.name, liveness: deps.liveness, label: 'the newest copy', @@ -4245,6 +4272,13 @@ async function reinstallOldApplications() { const appsInstalled = installedAppsRes.data; // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled) { + // An app whose removal is pending is not rebuilt: that removal would take + // the rebuilt app too. A later pass finds it again if it is still installed. + if (globalState.departingApps.has(installedApp.name)) { + log.info(`reinstallOldApplications - ${installedApp.name} is being removed, leaving it for a later pass`); + // eslint-disable-next-line no-continue + continue; + } // get current app specifications for the app name // if match found. Check if hash found. // if same, do nothing. if different remove and install. @@ -4365,79 +4399,12 @@ async function reinstallOldApplications() { // eslint-disable-next-line no-continue continue; } - log.warn('Updating from old application version, doing hard redeploy...'); log.warn(`REMOVAL REASON: App version upgrade - ${appSpecifications.name} upgrading from v${installedApp.version} to v${appSpecifications.version}`); - // eslint-disable-next-line global-require - const appUninstaller = require('./appUninstaller'); - // eslint-disable-next-line global-require - const appInstaller = require('./appInstaller'); - // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appSpecifications.name, null, true, false); - // connect to mongodb - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const appsQuery = { name: appSpecifications.name }; - const appsProjection = {}; - log.warn('Cleaning up database...'); - // eslint-disable-next-line no-await-in-loop - await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - const databaseStatus2 = { - status: 'Database cleaned', - }; - log.warn('Database cleaned'); - log.warn(databaseStatus2); - log.warn(`Compositions of application ${appSpecifications.name} uninstalled. Continuing with installation...`); - // composition removal done. Remove from installed apps and being installation + // A version 3 app has no components and a version 4 one has, so nothing of the + // old one carries over: the app is rebuilt from nothing. // eslint-disable-next-line no-await-in-loop - await appInstaller.checkAppRequirements(appSpecifications); // entire app - - // Register the app in database BEFORE creating Docker containers to prevent race condition - const isEnterprise = Boolean( - appSpecifications.version >= 8 && appSpecifications.enterprise, - ); - - const dbSpecs = JSON.parse(JSON.stringify(appSpecifications)); - - if (isEnterprise) { - dbSpecs.compose = []; - dbSpecs.contacts = []; - } - - // eslint-disable-next-line no-await-in-loop - const insertResult = await dbHelper.insertOneToDatabase(appsDatabase, localAppsInformation, dbSpecs); - if (!insertResult) { - throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name} during version upgrade reinstallation. Database insert returned undefined - likely duplicate key error or database failure. Aborting reinstallation to prevent orphaned Docker containers.`); - } - log.info(`Database entry created for ${appSpecifications.name} BEFORE component Docker container creation (version upgrade path)`); - - // Now install components - containers will be created but app is already in DB - let allComponentsBack = true; - const replaced = []; - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecifications.compose) { - log.warn(`Continuing Hard Redeployment of component ${appComponent.name}_${appSpecifications.name}...`); - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(config.fluxapps.redeploy.composedDelay * 1000); - // install the app - // eslint-disable-next-line no-await-in-loop - const outcome = await appInstaller.registerAppLocally(appSpecifications, appComponent); // component - if (outcome !== InstallOutcome.INSTALLED) { - log.error(`Component ${appComponent.name}_${appSpecifications.name} was not reinstalled (${outcome}). ${appSpecifications.name} is part-built; the app row describes every component, so the reconciler recreates what is missing.`); - allComponentsBack = false; - break; - } - replaced.push({ component: appComponent.name, hard: true }); - } - // Announced and restarted only if it is actually back. Saying so - // regardless is what turned a refused install into a node reporting - // an app it had taken apart. - if (allComponentsBack) { - log.warn(`Composed application ${appSpecifications.name} updated.`); - log.warn(`Restarting application ${appSpecifications.name}`); - // eslint-disable-next-line no-await-in-loop, no-use-before-define - await appDockerRestart(appSpecifications.name); - announceComponentsRedeployed(appSpecifications.name, replaced); - } + const outcome = await appRebuild.rebuildApp(appSpecifications); + log.warn(`Application ${appSpecifications.name} upgraded to v${appSpecifications.version}: ${outcome}`); } else if (appSpecifications.version <= 3) { if (appSpecifications.tiered) { const hddTier = `hdd${tier}`; @@ -4537,26 +4504,14 @@ async function reinstallOldApplications() { log.warn('Performing full hard redeploy to handle component changes...'); log.warn(`REMOVAL REASON: Component structure change (v8+) - ${appSpecifications.name} component count/names changed`); + // A hard redeploy: removed and installed again as a whole. One that does + // not bring the app back tells the network. // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appSpecifications.name, null, false, false); - const appRedeployResponse = messageHelper.createSuccessMessage('Application removed. Awaiting installation...'); - log.info(appRedeployResponse); - - // verify requirements - // eslint-disable-next-line no-await-in-loop - await appInstaller.checkAppRequirements(appSpecifications); - - // register - // eslint-disable-next-line no-await-in-loop - const outcome = await appInstaller.registerAppLocally(appSpecifications, undefined, null, false, true); + const outcome = await appRebuild.rebuildApp(appSpecifications); if (outcome === InstallOutcome.INSTALLED) { log.info(`Application ${appSpecifications.name} redeployed with new component structure`); } else { - // The removal above deleted the local row, and only a successful - // install writes it back. Nothing reconciles an app with no row, - // so this node has simply lost the instance until it is placed - // here again. - log.error(`Application ${appSpecifications.name} was removed for a component structure change and NOT reinstalled (${outcome}). This node no longer holds it and cannot recover it on its own.`); + log.error(`Application ${appSpecifications.name} was not redeployed for a component structure change (${outcome}).`); } // eslint-disable-next-line no-continue @@ -4608,21 +4563,13 @@ async function reinstallOldApplications() { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); const appsQuery = { name: appSpecifications.name }; - const appsProjection = {}; - log.warn('Cleaning up database...'); - // eslint-disable-next-line no-await-in-loop - await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - const databaseStatus2 = { - status: 'Database cleaned', - }; - log.warn('Database cleaned'); - log.warn(databaseStatus2); log.warn(`Compositions of application ${appSpecifications.name} uninstalled. Continuing with installation...`); - // composition removal done. Remove from installed apps and being installation // eslint-disable-next-line no-await-in-loop await appInstaller.checkAppRequirements(appSpecifications); // entire app - // Register the app in database BEFORE creating Docker containers to prevent race condition + // The app's row becomes the new specification in one write, before any + // container is created: until it lands the row still holds the old one, + // so the app always has a row describing it. const isEnterprise = Boolean( appSpecifications.version >= 8 && appSpecifications.enterprise, ); @@ -4635,11 +4582,8 @@ async function reinstallOldApplications() { } // eslint-disable-next-line no-await-in-loop - const insertResult = await dbHelper.insertOneToDatabase(appsDatabase, localAppsInformation, dbSpecs); - if (!insertResult) { - throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name} during composed app redeployment. Database insert returned undefined - likely duplicate key error or database failure. Aborting redeployment to prevent orphaned Docker containers.`); - } - log.info(`Database entry created for ${appSpecifications.name} BEFORE component Docker container creation (composed redeployment path)`); + await dbHelper.replaceOneInDatabase(appsDatabase, localAppsInformation, appsQuery, dbSpecs, { upsert: true }); + log.info(`Database entry of ${appSpecifications.name} replaced with its new specification BEFORE component Docker container creation (composed redeployment path)`); let allComponentsBack = true; const replaced = []; @@ -4697,6 +4641,14 @@ async function reinstallOldApplications() { } } catch (error) { log.error(error); + if (dbHelper.isDatabaseError(error)) { + // The components' data is still on disk, and a database that did not + // answer says nothing about the app: it is left for a later pass, never + // uninstalled with its data. + log.warn(`Redeployment of ${appSpecifications.name} stopped: the database did not answer (${error.message}). The app is not uninstalled.`); + // eslint-disable-next-line no-continue + continue; + } log.warn(`REMOVAL REASON: Redeployment error - ${appSpecifications.name} failed during redeployment: ${error.message}`); // eslint-disable-next-line no-await-in-loop await appUninstaller.removeAppLocally(appSpecifications.name, null, true, true, true); // remove entire app @@ -4714,6 +4666,7 @@ async function reinstallOldApplications() { // loop, on a path that can return or throw from several places, and a leaked // true would make every neighbour stand aside indefinitely. globalState.reinstallationOfOldAppsInProgress = false; + fluxEventBus.publish('reinstall:passEnded', {}); // ASKED ONCE THIS PASS IS OVER, because both of them uninstall and what holds them // apart is the node's install and removal flags: a removal the sweep attempts while // this pass holds those is refused, and goes onto the sweep's backoff rather than @@ -4779,8 +4732,13 @@ async function forceAppRemovals() { } const dockerAppsReported = await appQueryService.listAllApps(); - const dockerApps = dockerAppsReported.data; const installedAppsRes = await appQueryService.installedApps(); + // An app is an orphan only against a list of installed apps that was read. + if (dockerAppsReported.status !== 'success' || installedAppsRes.status !== 'success') { + log.warn('Unable to read the apps docker runs and the installed apps, skipping forceAppRemovals'); + return; + } + const dockerApps = dockerAppsReported.data; const appsInstalled = installedAppsRes.data; const dockerAppsNames = dockerApps.map((app) => { if (app.Names[0].startsWith('/zel')) { @@ -4821,8 +4779,9 @@ async function forceAppRemovals() { log.info(`${dApp} not found in locations for this IP (${localSocketAddr}), skipping broadcast`); } } catch (locationError) { - log.error(`Error checking app location for ${dApp}: ${locationError.message}`); - // Default to not broadcasting on error to avoid false positives + // A removal of a row that does not exist changes nothing on a peer. + shouldBroadcast = true; + log.error(`Error checking app location for ${dApp}, will broadcast removal: ${locationError.message}`); } // eslint-disable-next-line no-await-in-loop @@ -4882,12 +4841,169 @@ function staggerMs(places) { } /** - * Manages syncthing master/slave application coordination using FDM services. + * An app's g: components as the election decides them, in compose order, each + * with its operator stop lock. `free` is a component the election may run here: + * not locked, or locked and started by its operator. + * @param {object} installedApp Installed app specification, decrypted. + * @returns {Promise>} + */ +async function electedMembers(installedApp) { + const identifiers = mountParser.gComponentIdentifiers(installedApp); + const stops = await Promise.all(identifiers.map((identifier) => appsRuntimeState.operatorStopState(identifier))); + return identifiers.map((identifier, index) => ({ + identifier, + appId: dockerService.getAppIdentifier(identifier), + stop: stops[index], + free: !stops[index].stopped || stops[index].startRequested, + })); +} + +/** + * Whether this node holds a g: component: it runs here, a promotion to run it is + * in progress, or the reconciler has been asked to run it. + * @param {{identifier: string}} member + * @param {string[]} runningAppsNames Identifiers running on this node. + * @returns {boolean} + */ +function holdsMemberHere(member, runningAppsNames) { + return runningAppsNames.includes(member.identifier) + || primaryRole.inTransition(member.identifier) === primaryRole.Role.PROMOTING + || appReconciler.committedIdentifiers().includes(member.identifier); +} + +/** + * The g: components whose copy here is not yet one this node may run them on. A + * copy is ready when the syncthing monitor has marked it so, or its folder + * already sends here - which covers a ready mark lost with the process. + * @param {string} appName Global app name. + * @param {Array<{identifier: string, appId: string}>} members Components to check. + * @param {Map} receiveOnlyCache The syncthing monitor's per-folder state. + * @returns {Promise} identifiers of the components not ready + */ +async function unreadyMembers(appName, members, receiveOnlyCache) { + const unmarked = members.filter((member) => !receiveOnlyCache.get(member.appId)?.restarted); + if (!unmarked.length) return []; + let folders = []; + try { + const configured = await syncthingServiceModule.getConfigFolders(); + if (Array.isArray(configured)) folders = configured; + } catch (error) { + log.error(`masterSlaveApps: Failed to check syncthing folder status for ${appName}: ${error.message}`); + } + return unmarked + .filter((member) => { + // Syncthing syncs the entire appId folder (includes all subdirectories) + const sending = folders.some((folder) => folder.path === `${appsFolder}${member.appId}` && folder.type === 'sendreceive'); + if (sending) log.info(`masterSlaveApps: app:${appName} folder ${member.appId} is already in sendreceive mode, treating as ready`); + return !sending; + }) + .map((member) => member.identifier); +} + +/** + * Begins the start of each given g: component this node may run, in compose + * order: an operator-started one through its stop lock, any other as primary. + * @param {object[]} members The components to start. + * @param {string[]} runningAppsNames Identifiers running on this node. + * @returns {Promise} Whether a start was begun for any of them. + */ +async function startMembers(members, runningAppsNames) { + let begun = false; + // eslint-disable-next-line no-restricted-syntax + for (const member of members) { + if (!member.free || runningAppsNames.includes(member.identifier)) continue; + const started = member.stop.startRequested + // eslint-disable-next-line no-await-in-loop + ? await startOperatorStarted(member.identifier, member.appId) + : startAsPrimary(member.identifier, member.appId); + begun = started || begun; + } + return begun; +} + +/** + * Starts the g: components of an app that this node, as its primary, does not + * hold yet. + * + * A node holding none of them is taking the app over, and starts once every + * copy is ready: the primary runs them all, and one started on a partial copy + * publishes it as the truth. A node already holding some is the app's writer, + * and starts each missing one once that one's own copy is ready - a component + * the spec added, or one whose start failed. + * + * Either way it asks every other holder first. One that holds any g: component + * of the app is a second primary, and starting beside it splits the app across + * two nodes; it is stood down by FDM naming one of them, and the start waits for + * that. + * @param {string} appName Global app name. + * @param {object[]} members The app's g: components. + * @param {string[]} runningAppsNames Identifiers running on this node. + * @param {Map} receiveOnlyCache The syncthing monitor's per-folder state. + * @param {object} options + * @param {() => Promise} options.othersHold The PeerComponent state of + * the other holders. + * @param {boolean} [options.namedPrimary] FDM names this node the primary. A + * holder this node cannot rule out then does not stop the start: FDM found + * nothing running from where FDM stands, and a holder that was taking the app + * over stands down for a node FDM names that has decided it holds it. + * @returns {Promise} Whether a start was begun. + */ +async function startMissingMembers(appName, members, runningAppsNames, receiveOnlyCache, { othersHold, namedPrimary = false }) { + const { identifier } = members[0]; + const missing = members.filter((member) => member.free && !holdsMemberHere(member, runningAppsNames)); + if (!missing.length) return false; + const holdsHere = members.some((member) => holdsMemberHere(member, runningAppsNames)); + const unready = await unreadyMembers(appName, missing, receiveOnlyCache); + const toStart = holdsHere ? missing.filter((member) => !unready.includes(member.identifier)) : []; + if (!holdsHere && !unready.length) toStart.push(...missing); + if (unready.length) { + fluxEventBus.count('masterSlave:decision', identifier, 'notReady'); + log.info(`masterSlaveApps: app:${appName} is ${namedPrimary ? 'registered as primary on FDM' : 'held here'} but not ready yet (syncthing not synced${members.length > 1 ? ` for ${unready.join(', ')}` : ''}), skipping start for this cycle`); + } + if (!toStart.length) return false; + const others = await othersHold(); + if (others === PeerComponent.RUNNING || (!namedPrimary && others === PeerComponent.UNKNOWN)) { + fluxEventBus.count('masterSlave:decision', identifier, 'namedButHeldElsewhere'); + log.info(`masterSlaveApps: not starting app:${appName} that ${namedPrimary ? 'FDM names this node the primary of' : 'this node holds'} - a peer ${others === PeerComponent.RUNNING ? 'holds it' : 'could not be ruled out'}`); + return false; + } + const begun = await startMembers(toStart, runningAppsNames); + if (begun) log.info(`masterSlaveApps: starting docker component:${toStart.map((member) => member.identifier).join(', ')}`); + return begun; +} + +/** + * Lifts the stop lock of every g: component of an app its operator has started. + * @param {object[]} members The app's g: components. + * @param {string} outcome What the election decided. + * @returns {Promise} + */ +async function settleOperatorStarts(members, outcome) { + // eslint-disable-next-line no-restricted-syntax + for (const member of members) { + // eslint-disable-next-line no-await-in-loop + if (member.stop.startRequested) await settleOperatorStart(member.identifier, outcome); + } +} + +/** + * Withdraws this node's seed mark from the folder of every g: component of an + * app, once the election has decided another node runs them. + * @param {object[]} members The app's g: components. + * @param {string} outcome What the election decided. + */ +function withdrawSeedMarks(members, outcome) { + members.forEach((member) => withdrawSeedMark(member.appId, outcome)); +} + +/** + * One pass of the syncthing master/slave election, using FDM services. It + * returns when the pass is done and schedules nothing; startMasterSlaveApps + * runs it for the life of the process. * State (the busy lists, the receive-only cache) is read off globalStateParam at * the point of each decision, never taken as separate parameters: the getters - * hand out snapshots, so any list captured at call time is a photograph of that - * moment - and this function is invoked once at boot and re-invokes itself for - * the life of the process. + * hand out snapshots, so any list captured when the loop starts would be a + * photograph of that moment. * @param {object} globalStateParam - Global state module (busy lists, receive-only cache, run flags) * @param {Function} installedApps - Function to get installed apps * @param {Function} listRunningApps - Function to get running apps @@ -5016,9 +5132,6 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-restricted-syntax for (const installedApp of appsInstalled.data) { let fdmOk = true; - let identifier; - let needsToBeChecked = false; - let appId; const backupSkip = globalStateParam.backupInProgress.some((backupItem) => installedApp.name === backupItem); const restoreSkip = globalStateParam.restoreInProgress.some((backupItem) => installedApp.name === backupItem); if (backupSkip || restoreSkip) { @@ -5027,42 +5140,64 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-continue continue; } - // Check all g: mode apps, not just those in cache with restarted flag - // The cache tracks sync state, but shouldn't gate primary selection - identifier = mountParser.electedComponentIdentifier(installedApp); - if (identifier) { - appId = dockerService.getAppIdentifier(identifier); - needsToBeChecked = true; + // An app this node is removing, or has not yet confirmed it still holds, is + // neither promoted nor stood down: either would set its folder's type, and + // the reconciler starts none of its components. + if (globalStateParam.departingApps.has(installedApp.name)) { + fluxEventBus.count('masterSlave:decision', installedApp.name, 'skippedDeparting'); + // eslint-disable-next-line no-continue + continue; + } + if (globalStateParam.unconfirmedApps.has(installedApp.name)) { + fluxEventBus.count('masterSlave:decision', installedApp.name, 'skippedUnconfirmed'); + // eslint-disable-next-line no-continue + continue; } - if (needsToBeChecked) { - // One per pass per g: component, whatever the pass decides. Every start + // Every g: component of the app. The election decides the app: its primary + // runs all of them on one node, and every other holder runs none. Checked + // for all g: mode apps, not just those in cache with restarted flag - the + // cache tracks sync state, but shouldn't gate primary selection. + let members; + try { + // eslint-disable-next-line no-await-in-loop + members = await electedMembers(installedApp); + } catch (error) { + // An unread stop lock is not a lifted one: not a candidate this pass. + log.warn(`masterSlaveApps: ${installedApp.name}: a stop lock could not be read - excluded from primary election this pass: ${error.message}`); + // eslint-disable-next-line no-continue + continue; + } + if (members.length) { + // The anchor keys this app's election records and names it in the counters. + const { identifier } = members[0]; + // One per pass per g: app, whatever the pass decides. Every start // counts 'started', so passes counted from an event with no new start // is a decision not to start, taken that many times. fluxEventBus.count('masterSlave:decision', identifier, 'evaluated'); - // This node stopped the component in order to hand the app back, so it - // is not a candidate. Without this the election restores it within one - // cycle: the component is not running here, this node's own stale - // primary record is cleared below, no peer has picked it up yet, and the - // index-0 branch starts it again - every 30s, forever. Checked in the - // same place and for the same reason as operator-stopped, which is the - // other way a component this node holds is deliberately not running. - if (standingDown.has(identifier)) { - log.info(`masterSlaveApps: ${identifier} is standing down to be handed back - excluded from primary election`); + // This node stopped the app in order to hand it back, so it is not a + // candidate. Without this the election restores it within one cycle: the + // components are not running here, this node's own stale primary record + // is cleared below, no peer has picked it up yet, and the index-0 branch + // starts it again - every 30s, forever. Checked in the same place and for + // the same reason as operator-stopped, which is the other way a + // component this node holds is deliberately not running. + const handedBack = members.find((member) => standingDown.has(member.identifier)); + if (handedBack) { + log.info(`masterSlaveApps: ${handedBack.identifier} is standing down to be handed back - excluded from primary election`); // eslint-disable-next-line no-continue continue; } - // operator explicitly stopped this g: component; don't elect or act on it - // until they start it - // eslint-disable-next-line no-await-in-loop - const operatorStop = await appsRuntimeState.operatorStopState(identifier); - if (operatorStop.stopped && !operatorStop.startRequested) { + // operator explicitly stopped every g: component of the app; don't elect + // or act on it until they start one. A component stopped on its own stays + // down by its lock, and the election decides the rest. + if (!members.some((member) => member.free)) { // Outside the once-guard below on purpose: the log line reports the // state change, the counter reports every pass that honoured it, which // is what a test asserting "the election kept skipping it" needs. fluxEventBus.count('masterSlave:decision', identifier, 'operatorStopped'); if (!operatorStoppedNoted.has(identifier)) { operatorStoppedNoted.add(identifier); - log.info(`masterSlaveApps: ${identifier} is operator-stopped - excluded from primary election until it is started`); + log.info(`masterSlaveApps: ${members.map((member) => member.identifier).join(', ')} is operator-stopped - excluded from primary election until it is started`); } // eslint-disable-next-line no-continue continue; @@ -5070,14 +5205,16 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, operatorStoppedNoted.delete(identifier); // Started by its operator, and still locked: this pass decides it, and // the lock - which is what keeps this node holding the component - is - // lifted only by a decision. Starting it here, or finding it runs + // lifted only by a decision. Starting it here, or finding the app runs // elsewhere, lifts it; a pass that cannot tell leaves it for the next. - const resuming = operatorStop.startRequested; + const resumingMembers = members.filter((member) => member.stop.startRequested); + const resuming = resumingMembers.length > 0; if (resuming) { fluxEventBus.count('masterSlave:decision', identifier, 'operatorStartPending'); - log.info(`masterSlaveApps: ${identifier} was started by its operator - deciding it with the stop lock held`); + log.info(`masterSlaveApps: ${resumingMembers.map((member) => member.identifier).join(', ')} was started by its operator - deciding it with the stop lock held`); } - const start = async () => (resuming ? startOperatorStarted(identifier, appId) : startAsPrimary(identifier, appId)); + const runsHere = members.some((member) => runningAppsNames.includes(member.identifier)); + const start = () => startMembers(members, runningAppsNames); // eslint-disable-next-line no-await-in-loop await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.MASTERSLAVE_BEFORE_DECISION, identifier); // Get master IP from FDM using the new /appips endpoint @@ -5128,38 +5265,16 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, primaryElectionCheckedAt.set(identifier, Date.now()); if ((!ip)) { log.info(`masterSlaveApps: app:${installedApp.name} has currently no primary set`); - if (!runningAppsNames.includes(identifier)) { - // Check if app is ready (syncthing data is synced) before allowing it to become primary - let isReady = globalStateParam.receiveOnlySyncthingAppsCache.has(appId) && globalStateParam.receiveOnlySyncthingAppsCache.get(appId).restarted; - - // Fallback: If not in cache or not ready, check if syncthing folder is already in sendreceive mode - // This handles the case where folder is synced but cache was cleared/lost - if (!isReady) { - try { - // eslint-disable-next-line global-require - const syncthingService = require('../syncthingService'); - // eslint-disable-next-line no-await-in-loop - const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (Array.isArray(allSyncthingFolders)) { - // Syncthing syncs the entire appId folder (includes all subdirectories) - const folder = `${appsFolder}${appId}`; - // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders) { - if (syncthingFolder.path === folder && syncthingFolder.type === 'sendreceive') { - log.info(`masterSlaveApps: app:${installedApp.name} folder is already in sendreceive mode, treating as ready`); - isReady = true; - break; - } - } - } - } catch (error) { - log.error(`masterSlaveApps: Failed to check syncthing folder status for ${installedApp.name}: ${error.message}`); - } - } - - if (!isReady) { + if (!runsHere) { + // Check every g: component's copy is ready (syncthing data is + // synced) before allowing this node to become primary: the + // primary runs them all, and one started on a partial copy + // publishes it as the truth. + // eslint-disable-next-line no-await-in-loop + const unready = await unreadyMembers(installedApp.name, members.filter((member) => member.free), globalStateParam.receiveOnlySyncthingAppsCache); + if (unready.length) { fluxEventBus.count('masterSlave:decision', identifier, 'notReady'); - log.info(`masterSlaveApps: app:${installedApp.name} is not ready yet (syncthing not synced), skipping primary selection for this cycle`); + log.info(`masterSlaveApps: app:${installedApp.name} is not ready yet (syncthing not synced${members.length > 1 ? ` for ${unready.join(', ')}` : ''}), skipping primary selection for this cycle`); // eslint-disable-next-line no-continue continue; } @@ -5182,13 +5297,13 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // to clear this map. Drop it and let the normal paths decide. if (mastersRunningGSyncthingApps.has(identifier) && ipsMatch(mastersRunningGSyncthingApps.get(identifier), localSocketAddr) - && !runningAppsNames.includes(identifier)) { + && !runsHere) { mastersRunningGSyncthingApps.delete(identifier); log.info(`masterSlaveApps: cleared this node's own stale primary record for ${identifier} - it is not running here`); } - // Probe every other holder to see whether the g: component is - // already running, committed or held somewhere else. Every start + // Probe every other holder to see whether any g: component of the + // app is already running, committed or held somewhere else. Every start // path asks all of them, whatever this node's place in the order: // the order decides who goes first, not who can have started. A // node further down can hold a booking made before the one ahead @@ -5220,7 +5335,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // them together bounds the wait at one timeout regardless of peer // count. const probeCtx = { - appId, identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + appIds: members.map((member) => member.appId), identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', }; const checkPeersRunning = async () => { @@ -5231,13 +5346,13 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, } const state = await componentStateOnPeers(peers, probeCtx); if (state === PeerComponent.RUNNING) { - withdrawSeedMark(appId, 'a peer holds it'); - if (resuming) await settleOperatorStart(identifier, 'a peer holds it'); + withdrawSeedMarks(members, 'a peer holds it'); + if (resuming) await settleOperatorStarts(members, 'a peer holds it'); } return state; }; - if (resuming && operatorStop.asPrimary) { + if (resumingMembers.some((member) => member.stop.asPrimary)) { // Its owner started an app this node was the primary of when it // was stopped. The start returns it here, whatever this node's // place in the order: every other node that was stopped with it @@ -5286,10 +5401,10 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-await-in-loop const previousMasterState = await peerComponentState(previousMasterAddr, { ...probeCtx, label: 'previous primary' }); if (previousMasterState !== PeerComponent.NOT_RUNNING) { - if (previousMasterState === PeerComponent.RUNNING) withdrawSeedMark(appId, 'the previous primary runs it'); + if (previousMasterState === PeerComponent.RUNNING) withdrawSeedMarks(members, 'the previous primary runs it'); if (resuming && previousMasterState === PeerComponent.RUNNING) { // eslint-disable-next-line no-await-in-loop - await settleOperatorStart(identifier, 'the previous primary runs it'); + await settleOperatorStarts(members, 'the previous primary runs it'); } // Only THIS app is settled - the previous master still holds it, // so there is nothing to elect. Returning here would abandon the @@ -5376,7 +5491,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, log.info(`masterSlaveApps: holding the scheduled start of app:${installedApp.name} index: ${index} - a peer could not be ruled out`); } } else if (index > 0 && !mastersRunningGSyncthingApps.has(identifier) - && globalStateParam.receiveOnlySyncthingAppsCache.get(appId)?.designatedLeader) { + && members.some((member) => globalStateParam.receiveOnlySyncthingAppsCache.get(member.appId)?.designatedLeader)) { // The state machine's confirmed designated leader is the only // instance that can seed a newborn app: at genesis every other // instance is receiveonly with nothing to sync from, so serving @@ -5412,8 +5527,10 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // already-syncing branch and never reaches the election again. // Spent here, so it cannot take this node out of the stagger on // later primary losses. - const seedCache = globalStateParam.receiveOnlySyncthingAppsCache.get(appId); - if (seedCache) seedCache.designatedLeader = false; + members.forEach((member) => { + const seedCache = globalStateParam.receiveOnlySyncthingAppsCache.get(member.appId); + if (seedCache) seedCache.designatedLeader = false; + }); } } else if (index > 0 && !mastersRunningGSyncthingApps.has(identifier) && !timeTostartNewMasterApp.has(identifier)) { // Non-primary node with no history - schedule start based on @@ -5435,14 +5552,29 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // All other cases: don't start log.info(`masterSlaveApps: not starting app:${installedApp.name} index: ${index} - conditions not met for primary selection`); } - } else if (!appReconciler.hasControllerOpinion(identifier)) { + } else { // Running here with no primary named: this node holds the app, and // the election leaves it running. A named primary elsewhere moves // it to stopped through the standby branch below. - // eslint-disable-next-line no-await-in-loop - if (await appReconciler.adoptControllerDesired(identifier, 'running', 'masterSlave holder, no primary named')) { - fluxEventBus.count('masterSlave:decision', identifier, 'adopted'); + // eslint-disable-next-line no-restricted-syntax + for (const member of members) { + if (runningAppsNames.includes(member.identifier) && !appReconciler.hasControllerOpinion(member.identifier)) { + // eslint-disable-next-line no-await-in-loop + if (await appReconciler.adoptControllerDesired(member.identifier, 'running', 'masterSlave holder, no primary named')) { + fluxEventBus.count('masterSlave:decision', member.identifier, 'adopted'); + } + } } + // eslint-disable-next-line no-await-in-loop + await startMissingMembers(installedApp.name, members, runningAppsNames, globalStateParam.receiveOnlySyncthingAppsCache, { + othersHold: () => componentStateOnOtherHolders( + (name) => registryManagerModule.appLocation(name), + localSocketAddr, + { + appIds: members.map((member) => member.appId), identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + }, + ), + }); } } else { // This pass read a primary off FDM. Counted rather than published: @@ -5457,7 +5589,7 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // Every other holder, and the node FDM names as well: what a node that // has not decided its role asks before it does. const probeCtx = { - appId, identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', + appIds: members.map((member) => member.appId), identifier, appName: installedApp.name, liveness, logPrefix: 'masterSlaveApps', }; const othersHold = () => componentStateOnOtherHolders( (name) => registryManagerModule.appLocation(name), @@ -5465,89 +5597,64 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, probeCtx, { also: [ip] }, ); - const holdsHere = runningAppsNames.includes(identifier) - || primaryRole.inTransition(identifier) === primaryRole.Role.PROMOTING - || appReconciler.committedIdentifiers().includes(identifier); + const holdsHere = members.some((member) => holdsMemberHere(member, runningAppsNames)); if (!ipsMatch(localSocketAddr, ip)) { - withdrawSeedMark(appId, `FDM names ${ip} its primary`); + withdrawSeedMarks(members, `FDM names ${ip} its primary`); if (resuming) { // eslint-disable-next-line no-await-in-loop - await settleOperatorStart(identifier, `FDM names ${ip} its primary`); + await settleOperatorStarts(members, `FDM names ${ip} its primary`); } - // FDM names a node that runs the component, and also one that only + // FDM names a node that runs the app, and also one that only // holds it - by a stop lock, a promotion, or a sending folder - while // nothing runs it anywhere. This node stands down for a node that has - // decided it holds the component, or that cannot be shown not to have. + // decided it holds the app, or that cannot be shown not to have. // One that has not decided gives way to a node that has, and FDM // names this one once it runs. // eslint-disable-next-line no-await-in-loop if (holdsHere && await peerUndecided(`${extractIp(ip)}:${extractPort(ip)}`, { ...probeCtx, label: 'FDM primary' })) { - log.info(`masterSlaveApps: keeping component:${identifier} - FDM names ip:${ip}, which has not decided it holds it`); - // Stands down only the g: component on this node. Non-g siblings (e.g. a - // DB cluster component that needs all instances running) keep running. - } else if (primaryRole.standDown(identifier, appId, { running: runningAppsNames.includes(identifier) })) { - log.info(`masterSlaveApps: standing down as primary of component:${identifier} - FDM names ip:${ip}, localSocketAddr is: ${localSocketAddr}`); + log.info(`masterSlaveApps: keeping app:${installedApp.name} - FDM names ip:${ip}, which has not decided it holds it`); } else { - // A standby's folder receives and never sends: what it holds is the - // primary's, and anything written here is a local change for the - // primary's copy to overwrite. A folder that returned paused asks - // every other holder whether it runs the component, as the monitor - // does, and the node FDM names as well. - // eslint-disable-next-line no-await-in-loop - await primaryRole.holdAsStandby(identifier, appId, { othersHold }); - } - } else if (runningAppsNames.includes(identifier)) { - // The primary runs here, so its folder sends - whatever demoted it - // while the container was down. - // eslint-disable-next-line no-await-in-loop - await primaryRole.holdAsPrimary(identifier, appId); - } else if (ipsMatch(localSocketAddr, ip) && !runningAppsNames.includes(identifier)) { - // Check if app is ready (syncthing data is synced) before starting - let isReady = globalStateParam.receiveOnlySyncthingAppsCache.has(appId) && globalStateParam.receiveOnlySyncthingAppsCache.get(appId).restarted; - - // Fallback: If not in cache or not ready, check if syncthing folder is already in sendreceive mode - if (!isReady) { - try { - // eslint-disable-next-line global-require - const syncthingService = require('../syncthingService'); - // eslint-disable-next-line no-await-in-loop - const allSyncthingFolders = await syncthingService.getConfigFolders(); - if (Array.isArray(allSyncthingFolders)) { - // Syncthing syncs the entire appId folder (includes all subdirectories) - const folder = `${appsFolder}${appId}`; - // eslint-disable-next-line no-restricted-syntax - for (const syncthingFolder of allSyncthingFolders) { - if (syncthingFolder.path === folder && syncthingFolder.type === 'sendreceive') { - log.info(`masterSlaveApps: app:${installedApp.name} folder is already in sendreceive mode, treating as ready`); - isReady = true; - break; - } - } + // Stands down every g: component on this node, and only those. + // Non-g siblings (e.g. a DB cluster component that needs all + // instances running) keep running. One held down by its operator + // stays as its lock keeps it. + // eslint-disable-next-line no-restricted-syntax + for (const member of members.filter((each) => each.free)) { + if (primaryRole.standDown(member.identifier, member.appId, { running: runningAppsNames.includes(member.identifier) })) { + log.info(`masterSlaveApps: standing down as primary of component:${member.identifier} - FDM names ip:${ip}, localSocketAddr is: ${localSocketAddr}`); + } else { + // A standby's folder receives and never sends: what it holds is + // the primary's, and anything written here is a local change for + // the primary's copy to overwrite. A folder that returned paused + // asks every other holder whether it runs the app, as the + // monitor does, and the node FDM names as well. + // eslint-disable-next-line no-await-in-loop + await primaryRole.holdAsStandby(member.identifier, member.appId, { othersHold }); } - } catch (error) { - log.error(`masterSlaveApps: Failed to check syncthing folder status for ${installedApp.name}: ${error.message}`); } } - - // FDM also names a node that only holds the component while - // another has taken it over and is not running it yet, so a start - // here asks every other holder first and does not start beside one - // that holds it. A holder this node cannot rule out does not stop - // it: FDM found nothing running it from where FDM stands, and a - // holder that was taking it over stands down for a node FDM names - // that has decided it holds it. - // eslint-disable-next-line no-await-in-loop - const others = isReady && !holdsHere ? await othersHold() : PeerComponent.NOT_RUNNING; - if (!isReady) { - fluxEventBus.count('masterSlave:decision', identifier, 'notReady'); - log.info(`masterSlaveApps: app:${installedApp.name} is registered as primary on FDM but not ready yet (syncthing not synced), skipping start for this cycle`); - } else if (others === PeerComponent.RUNNING) { - fluxEventBus.count('masterSlave:decision', identifier, 'namedButHeldElsewhere'); - log.info(`masterSlaveApps: not starting app:${installedApp.name} that FDM names this node the primary of - a peer holds it`); - // eslint-disable-next-line no-await-in-loop - } else if (await start()) { - log.info(`masterSlaveApps: starting docker component:${identifier}`); + } else { + // FDM names this node, so it is the app's primary. Each g: + // component that runs here keeps its folder sending - whatever + // demoted it while the container was down - and each that does + // not is started. + // eslint-disable-next-line no-restricted-syntax + for (const member of members.filter((each) => runningAppsNames.includes(each.identifier))) { + // eslint-disable-next-line no-await-in-loop + await primaryRole.holdAsPrimary(member.identifier, member.appId); } + // FDM also names a node that only holds the app while another has + // taken it over and is not running it yet, so a start here asks + // every other holder first and does not start beside one that + // holds it. A holder this node cannot rule out does not stop it: + // FDM found nothing running it from where FDM stands, and a holder + // that was taking it over stands down for a node FDM names that has + // decided it holds it. + // eslint-disable-next-line no-await-in-loop + await startMissingMembers(installedApp.name, members, runningAppsNames, globalStateParam.receiveOnlySyncthingAppsCache, { + othersHold, + namedPrimary: true, + }); } } } @@ -5560,11 +5667,40 @@ async function masterSlaveApps(globalStateParam, installedApps, listRunningApps, // eslint-disable-next-line no-param-reassign globalStateParam.masterSlaveAppsRunning = false; fluxEventBus.count('masterSlave:cycles'); - await serviceHelper.delay(config.fluxapps.masterSlaveIntervalMs ?? 30 * 1000); - masterSlaveApps(globalStateParam, installedApps, listRunningApps, https); } } +/** + * Runs the master/slave election now, and again masterSlaveIntervalMs after + * each pass ends, so one pass runs at a time. + * @param {object} globalStateParam - Global state module (busy lists, receive-only cache, run flags) + * @param {Function} installedApps - Function to get installed apps + * @param {Function} listRunningApps - Function to get running apps + * @param {object} https - HTTPS module + * @returns {{ stop: Function }} stop() schedules no further pass + */ +function startMasterSlaveApps(globalStateParam, installedApps, listRunningApps, https) { + let timer = null; + let stopped = false; + + async function runPassThenSchedule() { + try { + await masterSlaveApps(globalStateParam, installedApps, listRunningApps, https); + } catch (error) { + log.error(`masterSlaveApps: ${error}`); + } + if (!stopped) timer = setTimeout(runPassThenSchedule, config.fluxapps.masterSlaveIntervalMs ?? 30 * 1000); + } + + function stop() { + stopped = true; + clearTimeout(timer); + } + + runPassThenSchedule(); + return { stop }; +} + module.exports = { createAppVolume, softRegisterAppLocally, @@ -5597,5 +5733,6 @@ module.exports = { checkAndRemoveEnterpriseAppsOnNonArcane, forceAppRemovals, masterSlaveApps, + startMasterSlaveApps, appDockerStart, }; diff --git a/ZelBack/src/services/appLifecycle/appInstaller.js b/ZelBack/src/services/appLifecycle/appInstaller.js index 6da5b03dab..4ada81c87d 100644 --- a/ZelBack/src/services/appLifecycle/appInstaller.js +++ b/ZelBack/src/services/appLifecycle/appInstaller.js @@ -27,13 +27,15 @@ const imageVerifier = require('../utils/imageVerifier'); const registryCredentialHelper = require('../utils/registryCredentialHelper'); const upnpService = require('../upnpService'); const globalState = require('../utils/globalState'); +const folderStates = require('../appMonitoring/folderStates'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const mountParser = require('../utils/mountParser'); const { ensureStignoreCovers } = require('../appSystem/syncthingIgnorePolicy'); const { findCommonArchitectures } = require('../utils/appUtilities'); const log = require('../../lib/log'); -const { localAppsInformation, scannedHeightCollection } = require('../utils/appConstants'); +const { localAppsInformation } = require('../utils/appConstants'); +const appPlacements = require('../appManagement/appPlacements'); const messageVerifier = require('../appMessaging/messageVerifier'); const { availableApps, getApplicationGlobalSpecifications } = require('../appDatabase/registryManager'); const hwRequirements = require('../appRequirements/hwRequirements'); @@ -124,7 +126,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re for (const port of appSpecifications.ports) { // eslint-disable-next-line no-await-in-loop const portResponse = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${appName}`); - if (portResponse === true) { + if (portResponse.mapped) { const portStatus = { status: `Port ${port} mapped OK`, }; @@ -134,7 +136,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (res.flush) res.flush(); } } else { - throw new Error(`Error: Port ${port} FAILed to map.`); + throw new Error(`Error: Port ${port} FAILed to map. ${portResponse.reason}`); } } } @@ -144,7 +146,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (isUPNP) { log.info('Custom port specified, mapping ports'); const portResponse = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(appSpecifications.port), `Flux_App_${appName}`); - if (portResponse === true) { + if (portResponse.mapped) { const portStatus = { status: `Port ${appSpecifications.port} mapped OK`, }; @@ -154,7 +156,7 @@ async function setupApplicationPorts(appSpecifications, appName, isComponent, re if (res.flush) res.flush(); } } else { - throw new Error(`Error: Port ${appSpecifications.port} FAILed to map.`); + throw new Error(`Error: Port ${appSpecifications.port} FAILed to map. ${portResponse.reason}`); } } } @@ -370,6 +372,17 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f } return InstallOutcome.BUSY; } + // A pending removal of the app removes whatever is installed in its place, so + // the app takes no install until that removal has finished. + if (globalState.departingApps.has(appSpecs.name)) { + const rStatus = messageHelper.createWarningMessage(`Flux App ${appSpecs.name} is being removed. Installation not possible`); + log.error(rStatus); + if (res) { + res.write(serviceHelper.ensureString(rStatus)); + if (res.flush) res.flush(); + } + return InstallOutcome.BUSY; + } globalState.installationInProgress = true; acquired = true; // A test install holds nothing: it writes the app's row like any other install @@ -531,7 +544,15 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f throw new Error(`CRITICAL: Failed to create database entry for ${appSpecifications.name}. Database insert returned undefined - likely duplicate key error or database failure. Aborting installation to prevent orphaned Docker containers.`); } log.info(`Database entry created for ${appSpecifications.name} BEFORE Docker container creation`); + // Installed from here: a peer deciding a cold-start seed waits on this node + // until its monitor has measured what it holds. + folderStates.installed(appSpecifications); claimed = true; + // The app's volumes are created next: this node's record of when it took + // the app starts now. The write must succeed, like the insert above: an + // install that went on without it would leave any earlier record of the + // app standing, dated before this volume existed. + if (!test) await appPlacements.recordPlacement(appSpecifications.name); const hddTier = `hdd${tier}`; const ramTier = `ram${tier}`; const cpuTier = `cpu${tier}`; @@ -664,16 +685,18 @@ async function attemptRegisterAppLocally(appSpecs, componentSpecs, res, test = f // costs the node every install, redeploy, spawn and reinstall pass it is offered // until FluxOS restarts. if (acquired) globalState.installationInProgress = false; - // Bound to the hold: a call that did not take it never wrote the app's row - // and never set the mark, so it has neither to take back. The mark is set - // under the hold with nothing awaited in between, so holding it is the same - // fact as having marked. + // The mark is bound to the hold: it is set under the hold with nothing awaited + // in between, so a call that took the hold has a mark to take back. The + // teardown is bound to the row: a call that never wrote one created nothing, + // and an app a refusal finds already installed is not its to remove. if (test && acquired) { - try { - await appUninstaller.removeAppLocally(appSpecs.name, null, true, false, false); - log.info(`Test cleanup completed for ${appSpecs.name}`); - } catch (cleanupError) { - log.error(`Error during test cleanup for ${appSpecs.name}: ${cleanupError.message}`); + if (claimed) { + try { + await appUninstaller.removeAppLocally(appSpecs.name, null, true, false, false); + log.info(`Test cleanup completed for ${appSpecs.name}`); + } catch (cleanupError) { + log.error(`Error during test cleanup for ${appSpecs.name}: ${cleanupError.message}`); + } } // Released after the teardown, not before: the row is what the announcement // reads, and it exists until the teardown has deleted it. @@ -1017,9 +1040,12 @@ async function installAppLocally(req, res) { if (!appname) { throw new Error('No Flux App specified'); } - let blockAllowance = config.fluxapps.ownerAppAllowance; - // needs to be logged in - const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); + // Placing an app on a node is not the node operator's call, for the same + // reason removing one is not: hosting an app is not owning it. The spawner + // decides placement; the owner decides everything else. An app not yet paid + // for is never placed here: it is tried out through testAppInstall, which + // tears it down again and never announces it. + const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (authorized) { // registerAppLocally refuses a concurrent removal or installation but not a // redeploy or the periodic reinstall pass, and those hold the node across a @@ -1030,41 +1056,8 @@ async function installAppLocally(req, res) { if (heldBy) { throw new Error(`Another application is undergoing ${heldBy}. Installation not possible.`); } - let appSpecifications; - // anyone can deploy temporary app - // favor temporary to launch test temporary apps - const tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(appname); - if (tempMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = tempMessage.appSpecifications; - // blockAllowance is used for future validation - // eslint-disable-next-line no-unused-vars - blockAllowance = config.fluxapps.temporaryAppAllowance; - } - if (!appSpecifications) { - // Placing a registered app on a node is not the node operator's call, for - // the same reason removing one is not: hosting an app is not owning it. - // This branch resolves an app BY NAME from the marketplace, the global - // registry or a permanent message, so an operator reaching it is choosing - // which customer's app runs on hardware they control - and for a g:/r: app - // the new instance syncs that customer's data down to it. The spawner - // decides placement; the owner decides everything else. - // - // The temporary-message branch above is untouched and stays open to any - // logged-in user: that is how an app is tested before it is registered, - // it is addressed by hash rather than by name, and it expires on its own - // (temporaryAppAllowance). - const ownerAuthorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); - if (!ownerAuthorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - return; - } - } - if (!appSpecifications) { - const allApps = await availableApps(); - appSpecifications = allApps.find((app) => app.name === appname); - } + const allApps = await availableApps(); + let appSpecifications = allApps.find((app) => app.name === appname); if (!appSpecifications) { // eslint-disable-next-line no-use-before-define appSpecifications = await getApplicationGlobalSpecifications(appname); @@ -1073,8 +1066,10 @@ async function installAppLocally(req, res) { if (!appSpecifications) { const permMessage = await messageVerifier.checkAppMessageExistence(appname); if (permMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = permMessage.appSpecifications; + // The message carries the hash and height its specification does not: + // the announcement names the app by that hash, and the expiry sweep + // removes a local row without a height. + appSpecifications = { ...permMessage.appSpecifications, hash: permMessage.hash, height: permMessage.height }; } } if (!appSpecifications) { @@ -1098,25 +1093,13 @@ async function installAppLocally(req, res) { appSpecifications.hash = hash; } - // get current height - const dbopen = dbHelper.databaseConnection(); - if (!appSpecifications.height && appSpecifications.height !== 0) { - // precaution for old temporary apps. Set up for custom test specifications. - const database = dbopen.db(config.database.daemon.database); - const query = { generalScannedHeight: { $gte: 0 } }; - const projection = { - projection: { - _id: 0, - generalScannedHeight: 1, - }, - }; - const result = await dbHelper.findOneInDatabase(database, scannedHeightCollection, query, projection); - if (!result) { - throw new Error('Scanning not initiated'); - } - const explorerHeight = serviceHelper.ensureNumber(result.generalScannedHeight); - appSpecifications.height = explorerHeight - config.fluxapps.blocksLasting + blockAllowance; // allow running for this amount of blocks + // An app's term runs from the height its payment was mined at, and the + // expiry sweep removes it when the term ends. A specification without one + // has no term to run, so it is not installed. + if (appSpecifications.height === undefined || appSpecifications.height === null) { + throw new Error(`Application Specifications of ${appname} carry no height`); } + const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); const appsQuery = {}; // all @@ -1193,6 +1176,20 @@ async function checkAppRequirements(appSpecs, skipGeolocation = false, skipStati return true; } +/** + * The owner whose key signs a pending app message: the owner a registration + * names, and for an update the owner of the app as registered, who signs the + * update whatever owner it names. + * @param {object} tempMessage A temporary app message. + * @returns {Promise} The signer's address, or null when it cannot be found. + */ +async function pendingMessageSigner(tempMessage) { + const isUpdate = tempMessage.type === 'fluxappupdate' || tempMessage.type === 'zelappupdate'; + if (!isUpdate) return tempMessage.appSpecifications?.owner ?? null; + const registered = await getApplicationGlobalSpecifications(tempMessage.appSpecifications?.name); + return registered?.owner ?? null; +} + /** * Test application installation - Similar to installAppLocally but for testing with reduced resource requirements * @param {object} req - Request object containing appname in params or query @@ -1210,7 +1207,6 @@ async function testAppInstall(req, res) { } log.info(`testAppInstall: ${appname}`); - let blockAllowance = config.fluxapps.ownerAppAllowance; // needs to be logged in const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); @@ -1223,15 +1219,21 @@ async function testAppInstall(req, res) { } let appSpecifications; - // anyone can deploy temporary app - // favor temporary to launch test temporary apps + // An app not yet paid for is tested by the owner whose key signs its + // message, addressed by its message hash. The flux team may test any. const tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(appname); if (tempMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = tempMessage.appSpecifications; - // blockAllowance is used for future validation - // eslint-disable-next-line no-unused-vars - blockAllowance = config.fluxapps.temporaryAppAllowance; + const caller = serviceHelper.ensureObject(authOf(req))?.zelid; + const signer = await pendingMessageSigner(tempMessage); + const mayTest = (Boolean(caller) && caller === signer) + || await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); + if (!mayTest) { + const errMessage = messageHelper.errUnauthorizedMessage(); + res.json(errMessage); + return; + } + // The hash is the message's, not the specification's. + appSpecifications = { ...tempMessage.appSpecifications, hash: tempMessage.hash }; } if (!appSpecifications) { @@ -1242,11 +1244,6 @@ async function testAppInstall(req, res) { // which customer's app runs on hardware they control - and for a g:/r: app // the new instance syncs that customer's data down to it. The spawner // decides placement; the owner decides everything else. - // - // The temporary-message branch above is untouched and stays open to any - // logged-in user: that is how an app is tested before it is registered, - // it is addressed by hash rather than by name, and it expires on its own - // (temporaryAppAllowance). const ownerAuthorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); if (!ownerAuthorized) { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -1268,8 +1265,7 @@ async function testAppInstall(req, res) { if (!appSpecifications) { const permMessage = await messageVerifier.checkAppMessageExistence(appname); if (permMessage) { - // eslint-disable-next-line prefer-destructuring - appSpecifications = permMessage.appSpecifications; + appSpecifications = { ...permMessage.appSpecifications, hash: permMessage.hash, height: permMessage.height }; } } @@ -1290,11 +1286,27 @@ async function testAppInstall(req, res) { } const daemonHeight = syncStatus.data.height; + // the formatter strips spec metadata; the hash and height must survive it + const { height, hash } = appSpecifications; appSpecifications = await checkAndDecryptAppSpecs(appSpecifications, { daemonHeight, owner: appSpecifications.owner, }); appSpecifications = specificationFormatter(appSpecifications); + appSpecifications.height = height; + appSpecifications.hash = hash; + } + + // An app this node holds is not tested here: the test would install over it. + const installed = await dbHelper.findOneInDatabase( + dbHelper.databaseConnection().db(config.database.appslocal.database), + localAppsInformation, + { name: appSpecifications.name }, + { projection: { _id: 0, name: 1 } }, + ); + if (installed) { + res.json(messageHelper.createErrorMessage(`Flux App ${appSpecifications.name} is installed on this node. Test it on a node that does not hold it.`)); + return; } // Test installation - similar to regular install but with test flag diff --git a/ZelBack/src/services/appLifecycle/appKill.js b/ZelBack/src/services/appLifecycle/appKill.js new file mode 100644 index 0000000000..3ce538db37 --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appKill.js @@ -0,0 +1,323 @@ +/** + * Kill every process of an app through its cgroups, without docker or the + * database, and say whether any of them can still run. + * + * An app's containers live in its slice (appSlice), found from the kernel alone. + * A container created before the slice existed lives in its runtime's scope, + * `docker-.scope`, outside any app's slice, and which app it belongs to is + * read from the runtime's own records on disk (readContainers), which stay + * readable while docker does not answer but not while its disk does not. The + * records are read once and passed to every call here; null stands for records + * that could not be read. Then no container outside the slices can be named, so + * those containers are killed and checked together (killOutsideSlices, + * stillRunnableOutsideSlices), and an app can be said to be unable to run only + * once none of them can. Where the records are is read from docker at boot; until + * then it is docker's default for the node's kind. Where each cgroup is, systemd + * says (appSlice.controlGroup). + * + * A process held in the kernel is not removed by a kill: it keeps the kill + * pending and dies the moment the kernel lets it go, without running the app's + * code again. So a cgroup whose every process has SIGKILL pending is as safe as + * an empty one. + */ + +const fs = require('fs').promises; +const path = require('path'); +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); +const appSlice = require('./appSlice'); +const { getAppIdentifier } = require('../dockerService'); +const { isArcane } = require('../utils/appConstants'); + +const SIGKILL_MASK = 1n << 8n; + +const DEFAULT_PATHS = { + cgroupRoot: '/sys/fs/cgroup', + procRoot: '/proc', + dockerRoot: isArcane ? '/dat/var/lib/docker' : '/var/lib/docker', +}; +let paths = { ...DEFAULT_PATHS }; + +/** + * @param {{cgroupRoot?: string, procRoot?: string, dockerRoot?: string}} options + * Where the cgroup filesystem, /proc and docker's data are; each one given replaces it. + */ +function usePaths(options) { + paths = { ...paths, ...options }; + appSlice.useCgroupRoot(paths.cgroupRoot); +} + +/** + * A file only root may read, read through sudo where FluxOS does not run as root. + * @param {string} file + * @returns {Promise} + */ +async function readAsRoot(file) { + const { stdout, error } = await serviceHelper.runCommand('cat', { runAsRoot: true, params: [file] }); + return error ? null : stdout; +} + +// The name FluxOS gives an app's containers carries the flux or zel prefix +// (dockerService.getAppIdentifier). +const APP_PREFIX = /^(flux|zel)/; + +/** + * The app a container belongs to: its runonflux.app label, which FluxOS sets on + * every container it creates; for a container made before the label, read from + * its name, `_` or `flux`. A label-less container + * of an app whose own name starts with flux reads as the name without it. + * @param {{name: string, app: string|null}} container + * @returns {string|null} Null for a container that is not an app's. + */ +function appOf({ name, app }) { + if (app) return app; + if (!APP_PREFIX.test(name)) return null; + if (name.includes('_')) return name.slice(name.indexOf('_') + 1); + return name.startsWith('flux') ? name.slice('flux'.length) : name; +} + +/** + * @param {{name: string, app: string|null}} container + * @param {string} appName + */ +function belongsTo({ name, app }, appName) { + if (app) return app === appName; + return name === getAppIdentifier(appName) || (APP_PREFIX.test(name) && name.endsWith(`_${appName}`)); +} + +/** + * @typedef {{id: string, name: string, app: string|null}} ContainerRecord + * The name without its leading '/', and the app its runonflux.app label names. + */ + +/** + * Every container docker has a record of, read from its records on disk. Nothing + * bounds the read: on a disk that does not answer it never returns. + * @returns {Promise} Null when the records cannot be listed. + */ +async function readContainers() { + const listing = await serviceHelper.runCommand('ls', { runAsRoot: true, params: ['-1', path.join(paths.dockerRoot, 'containers')] }); + if (listing.error) return null; + const ids = listing.stdout.split('\n').map((line) => line.trim()).filter((id) => /^[0-9a-f]{64}$/.test(id)); + const containers = []; + // eslint-disable-next-line no-restricted-syntax + for (const id of ids) { + // eslint-disable-next-line no-await-in-loop + const text = await readAsRoot(path.join(paths.dockerRoot, 'containers', id, 'config.v2.json')); + let parsed = null; + try { + parsed = JSON.parse(text ?? 'null'); + } catch { + parsed = null; + } + const name = parsed?.Name; + const app = parsed?.Config?.Labels?.['runonflux.app']; + if (typeof name === 'string') containers.push({ id, name: name.replace(/^\//, ''), app: typeof app === 'string' && app ? app : null }); + } + return containers; +} + +/** + * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers + * @returns {ContainerRecord[]} Empty when the records could not be read. + */ +function recordsOf(appName, containers) { + return (containers ?? []).filter((container) => belongsTo(container, appName)); +} + +/** + * The names of the app's containers. + * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers + * @returns {string[]} Empty when the records could not be read. + */ +function containersOf(appName, containers) { + return recordsOf(appName, containers).map(({ name }) => name); +} + +/** + * The scope of every container outside the app slices: directly under + * system.slice, where the runtime puts a container with no slice of its own. + * + * Such a container was created before FluxOS gave every app a slice + * (dockerService sets each container's CgroupParent); FluxOS creates none + * outside one. Once no node holds one, this, and the part docker's records play + * in finding an app's processes, can go. + * @returns {Promise} Their unit names; null when systemd does not answer. + */ +async function scopesOutsideSlices() { + const parent = await appSlice.controlGroup('system.slice'); + if (!parent) return null; + const entries = await fs.readdir(parent).catch(() => null); + return entries === null ? null : entries.filter((entry) => /^docker-[0-9a-f]{64}\.scope$/.test(entry)); +} + +/** + * The cgroups of the named units that exist now, each once: a scope inside a slice + * also named is read and killed with the slice. + * @param {string[]} units + * @returns {Promise} + */ +async function existingCgroups(units) { + const found = (await Promise.all(units.map(appSlice.controlGroup))).filter(Boolean); + const outermost = [...new Set(found)].filter((dir, _, all) => !all.some((other) => dir.startsWith(`${other}/`))); + const present = await Promise.all(outermost.map((dir) => fs.access(path.join(dir, 'cgroup.procs')).then(() => dir, () => null))); + return present.filter(Boolean); +} + +/** + * Every cgroup that holds a container of the app and exists now: its slice, and + * the scope of each of its containers the records name. + * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers + * @returns {Promise} + */ +async function cgroupsOf(appName, containers) { + const unit = appSlice.sliceName(appName); + const scopes = recordsOf(appName, containers).map(({ id }) => `docker-${id}.scope`); + return existingCgroups([...(unit ? [unit] : []), ...scopes]); +} + +/** + * The cgroup of every container outside the app slices that exists now. + * @returns {Promise} null when systemd does not answer. + */ +async function outsideSliceCgroups() { + const scopes = await scopesOutsideSlices(); + return scopes === null ? null : existingCgroups(scopes); +} + +/** + * Send SIGKILL to every process in each cgroup. + * @param {string} owner What the cgroups hold, for the log. + * @param {string[]} cgroups + */ +async function killIn(owner, cgroups) { + // eslint-disable-next-line no-restricted-syntax + for (const dir of cgroups) { + // eslint-disable-next-line no-await-in-loop + const { error } = await serviceHelper.runCommand('sh', { runAsRoot: true, params: ['-c', 'echo 1 > "$1"', 'sh', path.join(dir, 'cgroup.kill')] }); + if (error) log.error(`appKill - ${owner}: the kill could not be sent to ${dir}: ${error.message}`); + } +} + +/** + * Send SIGKILL to every process of the app. + * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers + * @returns {Promise} The cgroups it was sent to. + */ +async function kill(appName, containers) { + const cgroups = await cgroupsOf(appName, containers); + await killIn(appName, cgroups); + return cgroups; +} + +/** + * Send SIGKILL to every process of every container outside the app slices. + * @returns {Promise} The cgroups it was sent to; null when systemd + * does not say which containers are outside the slices. + */ +async function killOutsideSlices() { + const cgroups = await outsideSliceCgroups(); + if (cgroups !== null) await killIn('containers outside the app slices', cgroups); + return cgroups; +} + +/** + * @param {string} pid + * @returns {Promise} Whether the process has SIGKILL pending, or is gone. + */ +async function killPendingOrGone(pid) { + const status = await fs.readFile(path.join(paths.procRoot, pid, 'status'), 'utf8').catch(() => null); + if (status === null) return true; + return ['SigPnd', 'ShdPnd'].some((field) => { + const match = status.match(new RegExp(`^${field}:\\s*([0-9a-f]+)$`, 'm')); + return match ? (BigInt(`0x${match[1]}`) & SIGKILL_MASK) !== 0n : false; + }); +} + +/** + * Every process in a cgroup and the cgroups below it. A process lives only in a + * leaf, so an app's processes are in the scopes under its slice, not the slice. + * @param {string} dir + * @returns {Promise} Their pids. + */ +async function processesUnder(dir) { + const own = (await fs.readFile(path.join(dir, 'cgroup.procs'), 'utf8').catch(() => '')) + .split('\n').map((line) => line.trim()).filter(Boolean); + const entries = await fs.readdir(dir, { withFileTypes: true }).catch(() => []); + const below = await Promise.all(entries.filter((entry) => entry.isDirectory()).map((entry) => processesUnder(path.join(dir, entry.name)))); + return [...own, ...below.flat()]; +} + +/** + * Every process in the cgroups that is neither gone nor has SIGKILL pending. + * @param {string[]} cgroups + * @returns {Promise} Their pids. + */ +async function runnableIn(cgroups) { + const pids = (await Promise.all(cgroups.map(processesUnder))).flat(); + const settled = await Promise.all(pids.map(killPendingOrGone)); + return pids.filter((pid, i) => !settled[i]); +} + +/** + * Which processes of the containers outside the app slices could still run. + * @returns {Promise} Their pids; null when systemd does not say + * which containers are outside the slices. + */ +async function stillRunnableOutsideSlices() { + const cgroups = await outsideSliceCgroups(); + return cgroups === null ? null : runnableIn(cgroups); +} + +/** + * Which of the app's processes could still run its code: every one that is + * neither gone nor has SIGKILL pending. + * @param {string} appName + * @param {ContainerRecord[]|null} containers readContainers + * @returns {Promise} Their pids; empty when the app can never run + * again; null when none was found, the records could not be read, and a + * container outside the slices, which may be the app's, can still run or + * cannot be found. + */ +async function stillRunnable(appName, containers) { + const runnable = await runnableIn(await cgroupsOf(appName, containers)); + if (runnable.length || containers) return runnable; + return (await stillRunnableOutsideSlices())?.length === 0 ? [] : null; +} + +/** + * Every app with a slice or a container record on this node, found without + * docker or the database. + * @param {ContainerRecord[]|null} containers readContainers + * @returns {Promise} + */ +async function appsOnDisk(containers) { + const names = new Set(); + const parent = await appSlice.controlGroup('flux.slice'); + const slices = parent ? await fs.readdir(parent).catch(() => []) : []; + slices.filter((entry) => /^flux-.+\.slice$/.test(entry)) + .forEach((entry) => names.add(entry.slice('flux-'.length, -'.slice'.length).replace(/\\x2d/g, '-'))); + (containers ?? []).map(appOf).filter(Boolean).forEach((name) => names.add(name)); + return [...names]; +} + +/** Restore the default paths. */ +function reset() { + usePaths(DEFAULT_PATHS); +} + +module.exports = { + readContainers, + containersOf, + kill, + killOutsideSlices, + stillRunnable, + stillRunnableOutsideSlices, + appsOnDisk, + usePaths, + reset, +}; diff --git a/ZelBack/src/services/appLifecycle/appNetworkLinker.js b/ZelBack/src/services/appLifecycle/appNetworkLinker.js index 8c18bf2c09..a73565f30c 100644 --- a/ZelBack/src/services/appLifecycle/appNetworkLinker.js +++ b/ZelBack/src/services/appLifecycle/appNetworkLinker.js @@ -26,6 +26,30 @@ const dockerService = require('../dockerService'); const log = require('../../lib/log'); const { localAppsInformation, APP_NAME_REGEX } = require('../utils/appConstants'); +// The delays before a sweep whose installed-apps read failed runs again, by +// attempt; the last repeats until the read succeeds. One retry is pending per +// sweep at a time. +const READ_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; +const pendingRetries = new Map(); + +/** + * Runs `sweep(attempt + 1)` once the delay for `attempt` has passed, unless a + * retry of the same sweep is pending already. + * @param {string} key Names the sweep. + * @param {function(number): Promise} sweep + * @param {number} attempt + */ +function retrySweep(key, sweep, attempt) { + if (pendingRetries.has(key)) return; + const delay = READ_RETRY_MS[Math.min(attempt, READ_RETRY_MS.length - 1)]; + const timer = setTimeout(() => { + pendingRetries.delete(key); + sweep(attempt + 1).catch((error) => log.error(`${key}: ${error.message}`)); + }, delay); + if (timer.unref) timer.unref(); + pendingRetries.set(key, timer); +} + /** * Parses the `networkWith:[...]` token out of an app description. * @@ -127,19 +151,22 @@ async function connectComponentToLinkedApps(componentContainerName, fullAppSpecs /** * After an app's private network is (re)created, reconnects every locally * installed app that is networked with it back onto that network. Best-effort — - * never throws, so a redeploy is not aborted by a reconnect hiccup. + * never throws, so a redeploy is not aborted by a reconnect hiccup. When the + * installed apps cannot be read it runs again later (READ_RETRY_MS). * * @param {string} appName - the app whose network was (re)created + * @param {number} [attempt] - how many reads failed before this one * @returns {Promise} */ -async function reconnectLinkedApps(appName) { +async function reconnectLinkedApps(appName, attempt = 0) { let installedApps; try { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); installedApps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, {}, { projection: { _id: 0 } }); } catch (error) { - log.error(`reconnectLinkedApps: failed to read installed apps for ${appName}: ${error.message}`); + log.error(`reconnectLinkedApps: failed to read installed apps for ${appName}, retrying later: ${error.message}`); + retrySweep(`reconnectLinkedApps ${appName}`, (next) => reconnectLinkedApps(appName, next), attempt); return; } @@ -174,18 +201,21 @@ async function reconnectLinkedApps(appName) { /** * Boot-time sweep: ensures every installed app that declares network links is - * attached to each linked app's network. Idempotent and best-effort. + * attached to each linked app's network. Idempotent and best-effort. When the + * installed apps cannot be read it runs again later (READ_RETRY_MS). * + * @param {number} [attempt] - how many reads failed before this one * @returns {Promise} */ -async function reconcileAllAppNetworkLinks() { +async function reconcileAllAppNetworkLinks(attempt = 0) { let installedApps; try { const dbopen = dbHelper.databaseConnection(); const appsDatabase = dbopen.db(config.database.appslocal.database); installedApps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, {}, { projection: { _id: 0 } }); } catch (error) { - log.error(`reconcileAllAppNetworkLinks: failed to read installed apps: ${error.message}`); + log.error(`reconcileAllAppNetworkLinks: failed to read installed apps, retrying later: ${error.message}`); + retrySweep('reconcileAllAppNetworkLinks', reconcileAllAppNetworkLinks, attempt); return; } diff --git a/ZelBack/src/services/appLifecycle/appRebuild.js b/ZelBack/src/services/appLifecycle/appRebuild.js new file mode 100644 index 0000000000..bdaae867e4 --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appRebuild.js @@ -0,0 +1,54 @@ +/** + * A hard redeploy: an app removed from this node, data and all, and installed again + * at a specification, the whole app at once. The one way an app is rebuilt from + * nothing; the callers decide when, and hold the node while it runs. + */ +const log = require('../../lib/log'); +const fluxEventBus = require('../utils/fluxEventBus'); +const appUninstaller = require('./appUninstaller'); +const appInstaller = require('./appInstaller'); +const { RemovalOutcome } = require('../utils/removalOutcome'); +const { InstallOutcome } = require('../utils/installOutcome'); + +/** + * Remove the app and install it again at appSpecs. + * + * Nothing is installed unless the removal finished. A node busy with another removal + * leaves the app as it was; a removal that stopped part way is finished by the + * uninstaller, which removes the whole app and tells the network. Once the removal + * has finished this node holds nothing of the app, so an install that does not + * complete is told to the network, once, here. + * @param {object} appSpecs The specification to install. + * @param {object} [res] A response stream the caller holds. + * @returns {Promise} The InstallOutcome; BUSY when another removal held the + * node, FAILED when the removal did not finish. + */ +async function rebuildApp(appSpecs, res) { + const removal = await appUninstaller.removeAppLocally(appSpecs.name, res, false, false); + if (removal !== RemovalOutcome.REMOVED && removal !== RemovalOutcome.NOT_INSTALLED) { + log.warn(`appRebuild - ${appSpecs.name} was not removed (${removal}), so it is not reinstalled`); + const outcome = removal === RemovalOutcome.BUSY ? InstallOutcome.BUSY : InstallOutcome.FAILED; + fluxEventBus.publish('app:rebuilt', { name: appSpecs.name, outcome }); + return outcome; + } + + let outcome; + try { + await appInstaller.checkAppRequirements(appSpecs); + // the installer's own teardown says nothing: whether to tell the network is decided below + outcome = await appInstaller.registerAppLocally(appSpecs, undefined, res, false, false); + } catch (error) { + log.error(`appRebuild - ${appSpecs.name} was not reinstalled: ${error.message}`); + outcome = InstallOutcome.FAILED; + } + + if (outcome !== InstallOutcome.INSTALLED && removal === RemovalOutcome.REMOVED) { + log.warn(`appRebuild - ${appSpecs.name} was removed and not reinstalled (${outcome}); telling the network`); + await appUninstaller.announceRemoval(appSpecs.name) + .catch((error) => log.error(`appRebuild - the removal of ${appSpecs.name} was not announced: ${error.message}`)); + } + fluxEventBus.publish('app:rebuilt', { name: appSpecs.name, outcome }); + return outcome; +} + +module.exports = { rebuildApp }; diff --git a/ZelBack/src/services/appLifecycle/appSlice.js b/ZelBack/src/services/appLifecycle/appSlice.js new file mode 100644 index 0000000000..9740a91eeb --- /dev/null +++ b/ZelBack/src/services/appLifecycle/appSlice.js @@ -0,0 +1,124 @@ +/** + * Every app's containers live in their own systemd slice, under flux.slice, so + * the app's processes are found and killed from its name alone, without docker + * or the database. Under contention the apps together share the machine + * against the system services as one group, so they cannot starve FluxOS's own + * dependencies. + * + * A slice outlives its last container until it is stopped. Removing an app + * stops its slice, and boot stops every slice left empty. + * + * Where a unit's cgroup is, systemd says: its tree starts wherever systemd itself + * runs, which is the filesystem's root on a host and below it when systemd runs + * in a container. + */ + +const fs = require('fs').promises; +const path = require('path'); +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); + +const PARENT = 'flux.slice'; +const APP_NAME = /^[A-Za-z0-9-]+$/; +// systemd answers from memory; a question it has not answered by then gets no answer +const SYSTEMD_ANSWER_MS = 10 * 1000; + +let cgroupRoot = '/sys/fs/cgroup'; + +/** + * @param {string} root Where the cgroup filesystem is mounted. + */ +function useCgroupRoot(root) { + cgroupRoot = root; +} + +/** + * The app's slice unit. systemd reads each '-' in a slice name as one level of + * nesting, so the name escapes it the way systemd-escape does: unescaped, + * `my-app` would land inside `my`'s slice. + * @param {string} appName + * @returns {string|null} Null for a name no app can have. + */ +function sliceName(appName) { + if (typeof appName !== 'string' || !APP_NAME.test(appName)) return null; + return `flux-${appName.replace(/-/g, '\\x2d')}.slice`; +} + +/** + * The unit's cgroup directory, as systemd reports it. + * @param {string} unit Any systemd unit: a slice, a scope, a service. + * @returns {Promise} Null when systemd has no such unit, or no answer. + */ +async function controlGroup(unit) { + const { stdout, error } = await serviceHelper.runCommand('systemctl', { + params: ['show', '-p', 'ControlGroup', '--value', unit], timeout: SYSTEMD_ANSWER_MS, logError: false, + }); + const group = (stdout ?? '').trim(); + return error || !group ? null : path.join(cgroupRoot, group); +} + +/** + * @param {string} unit + * @returns {Promise} Whether any process lives in the slice, or + * null when the slice does not exist. + */ +async function isPopulated(unit) { + const dir = await controlGroup(unit); + const events = dir ? await fs.readFile(path.join(dir, 'cgroup.events'), 'utf8').catch(() => null) : null; + if (events === null) return null; + return /^populated 1$/m.test(events); +} + +/** + * Stop a slice that holds no process. A slice in use is left alone. + * + * The stop is queued and not waited for: an empty slice has nothing to stop, and + * systemd's own bus timeout bounds the request. + * @param {string} unit + * @returns {Promise} True when systemd took the stop. + */ +async function stopIfEmpty(unit) { + if (await isPopulated(unit) !== false) return false; + const { error } = await serviceHelper.runCommand('systemctl', { runAsRoot: true, params: ['stop', '--no-block', unit] }); + if (error) { + log.error(`appSlice - ${unit} could not be stopped: ${error.message}`); + return false; + } + return true; +} + +/** + * Stop the app's slice, once its containers are gone. + * @param {string} appName + */ +async function release(appName) { + const unit = sliceName(appName); + if (unit) await stopIfEmpty(unit); +} + +/** + * Stop every app slice that holds no process: those whose app was removed + * while FluxOS was not running to stop them. + * @returns {Promise} How many were stopped. + */ +async function stopEmptySlices() { + const parent = await controlGroup(PARENT); + const entries = parent ? await fs.readdir(parent).catch(() => []) : []; + const slices = entries.filter((entry) => entry.startsWith('flux-') && entry.endsWith('.slice')); + let stopped = 0; + // eslint-disable-next-line no-restricted-syntax + for (const unit of slices) { + // eslint-disable-next-line no-await-in-loop + if (await stopIfEmpty(unit)) stopped += 1; + } + if (stopped) log.info(`appSlice - stopped ${stopped} empty app slice(s)`); + return stopped; +} + +module.exports = { + sliceName, + controlGroup, + release, + stopEmptySlices, + useCgroupRoot, +}; diff --git a/ZelBack/src/services/appLifecycle/appSpawner.js b/ZelBack/src/services/appLifecycle/appSpawner.js index d3cd50f53b..3102e3fdd7 100644 --- a/ZelBack/src/services/appLifecycle/appSpawner.js +++ b/ZelBack/src/services/appLifecycle/appSpawner.js @@ -700,7 +700,7 @@ async function trySpawningGlobalApplication() { // Note: User-blocked port check happens earlier (line ~353) before Docker Hub calls // Check if ports are publicly available - critical for proper Flux network operation - const portVerdict = await portManager.checkInstallingAppPortAvailable(appPorts); + const portVerdict = await portManager.checkInstallingAppPortAvailable(appPorts, appSpecifications.name); if (portVerdict.ok === false) { log.error(`trySpawningGlobalApplication - Some of application ports of ${appSpecifications.name} are not available publicly. Installation aborted.`); // The cause lives in portManager, which says which port and which peers; @@ -1178,6 +1178,12 @@ async function trySpawningGlobalApplication() { return isEnterprise ? 0 : delayTime; } catch (error) { log.error(error); + // A database that did not answer says nothing about the app: the next pass + // tries it again. + if (appHash && dbHelper.isDatabaseError(error)) { + globalState.trySpawningGlobalAppCache.delete(appHash); + return shortDelayTime || 5 * 60 * 1000; + } if (appHash && !globalState.spawnErrorsLongerAppCache.has(appHash) && !globalState.trySpawningGlobalAppCache.has(appHash)) { log.info(`trySpawningGlobalApplication - Adding app hash ${appHash} to trySpawningGlobalAppCache due to pre-install error`); globalState.trySpawningGlobalAppCache.set(appHash, '', { ttl: FluxCacheManager.oneHour * 6 }); diff --git a/ZelBack/src/services/appLifecycle/appStartupManager.js b/ZelBack/src/services/appLifecycle/appStartupManager.js index b87e53151f..81916d8979 100644 --- a/ZelBack/src/services/appLifecycle/appStartupManager.js +++ b/ZelBack/src/services/appLifecycle/appStartupManager.js @@ -16,12 +16,18 @@ const registryManager = require('../appDatabase/registryManager'); const appReconciler = require('../appMonitoring/appReconciler'); const appUninstaller = require('./appUninstaller'); const appNetworkLinker = require('./appNetworkLinker'); +const appSlice = require('./appSlice'); +const nodeStatusMonitor = require('../appMonitoring/nodeStatusMonitor'); +const appPlacements = require('../appManagement/appPlacements'); +const { TIMED_OUT } = require('../utils/withinDeadline'); const portManager = require('../appNetwork/portManager'); const globalState = require('../utils/globalState'); const fluxEventBus = require('../utils/fluxEventBus'); const nodeConfirmationService = require('../nodeConfirmationService'); const { localAppsInformation, SIGTERM_EXPIRY_MS, RUNNING_EXPIRY_MS } = require('../utils/appConstants'); const { getNonGComponentIdentifiers, parseContainerName, appHasValidLocationOnNode } = require('../utils/appUtilities'); +const { RemovalOutcome } = require('../utils/removalOutcome'); +const { untilSucceeds, BOOT_STEP_RETRY_MS } = require('../utils/untilSucceeds'); const SYNC_TIMEOUT_MS = config.system.bootSyncTimeoutMs ?? 300000; @@ -52,19 +58,14 @@ async function awaitWithin(work, timeoutMs, reason) { * @returns {Promise} Array of installed app specifications */ async function getInstalledAppsFromDb() { - try { - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const appsQuery = {}; - const appsProjection = { - projection: { _id: 0 }, - }; - const apps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); - return apps || []; - } catch (error) { - log.error(`appStartupManager - Error getting installed apps: ${error.message}`); - return []; - } + const dbopen = dbHelper.databaseConnection(); + const appsDatabase = dbopen.db(config.database.appslocal.database); + const appsQuery = {}; + const appsProjection = { + projection: { _id: 0 }, + }; + const apps = await dbHelper.findInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); + return apps || []; } /** @@ -118,12 +119,97 @@ async function openInstalledAppPorts() { } } + +/** + * Removes an app whose location this node no longer holds. The network has dropped + * the record, so the removal tells it nothing. A removal that does not finish holds + * the app as departing - never started by the reconciler, never announced as + * running - and is run again until it does. + * @param {string} appName + * @returns {Promise} Whether the app is gone now. + */ +async function removeAppBootMayNotStart(appName) { + const outcome = await appUninstaller.removeAppLocally(appName, null, true, true, false); + if (outcome === RemovalOutcome.REMOVED || outcome === RemovalOutcome.NOT_INSTALLED) return true; + appUninstaller.finishRemoval(appName, true, false); + return false; +} + +/** + * Hands the components boot starts to the reconciler. + * @param {string[]} componentsToStart + * @returns {void} + */ +function startAtBoot(componentsToStart) { + // the reconciler actuates once the boot gate opens (daemon/DB ready) and applies + // desired-state + restart policy + // eslint-disable-next-line no-restricted-syntax + for (const identifier of componentsToStart) { + appReconciler.enqueue(identifier); + log.info(`appStartupManager - Queued ${identifier} for reconcile`); + } +} + +/** + * Whether this node holds an unexpired location record for an app. + * @param {string} appName + * @param {string} localSocketAddr + * @returns {Promise} + * @throws {Error} When the records cannot be read. + */ +async function readLocation(appName, localSocketAddr) { + const valid = await appHasValidLocationOnNode(appName, localSocketAddr); + if (valid === null) throw new Error(`the location record of ${appName} could not be read`); + return valid; +} + +/** + * Decides an app held in globalState.unconfirmedApps once its location record can + * be read, reading it every BOOT_STEP_RETRY_MS until then. A valid record releases + * the app and starts the components boot starts; any other answer removes it, the + * app held until the removal has finished or holds it as departing. An app whose + * hold has ended by then - uninstalled meanwhile, perhaps installed again since - + * is not this check's to decide, and it does nothing. + * @param {string} appName + * @param {string[]} componentsToStart + * @param {string} localSocketAddr + * @param {symbol} hold The hold this check decides, as boot set it. + * @returns {Promise} + */ +async function confirmHeldApp(appName, componentsToStart, localSocketAddr, hold) { + const isHeld = () => globalState.unconfirmedApps.get(appName) === hold; + await serviceHelper.delay(BOOT_STEP_RETRY_MS); + const valid = await untilSucceeds(`Location check of ${appName}`, () => readLocation(appName, localSocketAddr)); + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.BOOT_BEFORE_LOCATION_DECISION, appName); + // Nothing awaits between this check and the decision below, so the hold cannot + // end in between. + if (!isHeld()) { + log.info(`appStartupManager - App ${appName}: its hold ended before its location record was read, nothing to decide`); + fluxEventBus.publish('boot:appConfirmed', { name: appName, valid, released: true }); + return; + } + fluxEventBus.publish('boot:appConfirmed', { name: appName, valid }); + if (valid) { + log.info(`appStartupManager - App ${appName}: its location record is valid, starting it`); + globalState.unconfirmedApps.delete(appName); + startAtBoot(componentsToStart); + return; + } + log.warn(`appStartupManager - App ${appName}: no valid location record for this node (${localSocketAddr}), removing locally`); + try { + await removeAppBootMayNotStart(appName); + } finally { + if (isHeld()) globalState.unconfirmedApps.delete(appName); + } +} /** * Reconcile local app state against the network on boot. For each stopped * container, checks whether this node still has a valid location record. * - Valid location, non-g components: started. * - Valid location, g: components: left stopped (managed by masterSlaveApps). * - Expired/missing location: app removed locally (node was reassigned). + * - Unreadable location: app held stopped until the record can be read, then + * decided as above (confirmHeldApp). * @returns {Promise} Results of the reconciliation */ async function reconcileAppsOnBoot() { @@ -134,6 +220,7 @@ async function reconcileAppsOnBoot() { appsSkippedGMode: [], appsSkippedNoSpec: [], appsRemoved: [], + appsUnconfirmed: [], // location unreadable: held stopped until it can be read appsFailed: [], }; @@ -227,17 +314,26 @@ async function reconcileAppsOnBoot() { if (localSocketAddr) { // eslint-disable-next-line no-await-in-loop const hasValidLocation = await appHasValidLocationOnNode(appName, localSocketAddr); + if (hasValidLocation === null) { + // Whether the network placed it elsewhere while this node was down is not + // known, so it is neither started nor removed until it is. + log.warn(`appStartupManager - App ${appName}: its location record could not be read, held stopped until it can be`); + const hold = Symbol(appName); + globalState.unconfirmedApps.set(appName, hold); + fluxEventBus.publish('boot:appUnconfirmed', { name: appName }); + confirmHeldApp(appName, componentsToStart, localSocketAddr, hold) + .catch((error) => log.error(`appStartupManager - App ${appName}: the location check stopped: ${error.message}`)); + results.appsUnconfirmed.push(appName); + // eslint-disable-next-line no-continue + continue; + } if (!hasValidLocation) { - log.warn(`appStartupManager - App ${appName} no longer has a valid location record for this node (${localSocketAddr}), removing locally`); - try { - // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(appName, null, true, true, false); - results.appsRemoved.push(appName); - log.info(`appStartupManager - App ${appName} removed locally (was reassigned to another node)`); - } catch (removeError) { - log.error(`appStartupManager - Failed to remove app ${appName}: ${removeError.message}`); - results.appsFailed.push({ app: appName, error: removeError.message }); - } + // placed elsewhere while this node was down: removed, never started + log.warn(`appStartupManager - App ${appName}: no valid location record for this node (${localSocketAddr}), removing locally`); + // eslint-disable-next-line no-await-in-loop + const removed = await removeAppBootMayNotStart(appName); + if (removed) results.appsRemoved.push(appName); + else results.appsFailed.push({ app: appName, error: 'not removed; held stopped until it is' }); // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(2000); // eslint-disable-next-line no-continue @@ -253,13 +349,7 @@ async function reconcileAppsOnBoot() { log.info(`appStartupManager - App ${appName} has valid location, starting app`); } - // hand each component to the reconciler; it actuates once the boot gate - // opens (daemon/DB ready) and applies desired-state + restart policy - // eslint-disable-next-line no-restricted-syntax - for (const identifier of componentsToStart) { - appReconciler.enqueue(identifier); - log.info(`appStartupManager - Queued ${identifier} for reconcile`); - } + startAtBoot(componentsToStart); if (isPartial) { results.appsPartiallyStarted.push(appName); @@ -274,6 +364,7 @@ async function reconcileAppsOnBoot() { + `Apps started: ${results.appsStarted.length}, ` + `Apps partially started (mixed g:): ${results.appsPartiallyStarted.length}, ` + `Apps removed (expired location): ${results.appsRemoved.length}, ` + + `Apps held (location unreadable): ${results.appsUnconfirmed.length}, ` + `Apps skipped (g: mode): ${results.appsSkippedGMode.length}, ` + `Apps skipped (no spec): ${results.appsSkippedNoSpec.length}, ` + `Apps failed: ${results.appsFailed.length}`, @@ -296,16 +387,36 @@ async function removeAllApps(reason) { if (installedAppsRes.status === 'success') { for (const app of installedAppsRes.data) { log.warn(`REMOVAL REASON: ${reason} - removing ${app.name}`); + // One removal that never returns must not hold boot, or every app after it. // eslint-disable-next-line no-await-in-loop - await appUninstaller.removeAppLocally(app.name, null, true, false, true); + const outcome = await nodeStatusMonitor.removeWithinDeadline(appUninstaller.removeAppLocally, app.name, true); + if (outcome === TIMED_OUT) log.error(`appStartupManager - ${app.name} was not removed within ${config.fluxapps.appRemovalDeadlineMs / 1000}s, moving on`); + else if (outcome === nodeStatusMonitor.STILL_REMOVING) log.warn(`appStartupManager - ${app.name} is still being removed, moving on`); } } } +/** + * Remove the placement records of apps not installed here, on every boot + * whichever way it goes on. Run again until it succeeds. + * @returns {Promise} + */ +async function tidyPlacementsOnBoot() { + await untilSucceeds('Removing stray placement records', async () => { + const installed = await getInstalledAppsFromDb(); + await appPlacements.removeStrayPlacements(installed.map((app) => app.name)); + }); +} + async function manageAppsOnBoot(bootContext) { try { + await tidyPlacementsOnBoot(); + // ahead of every container start, so no slice in use can be empty here + await appSlice.stopEmptySlices(); if (bootContext.firstBoot) { log.info('appStartupManager - First boot (no heartbeat history), waiting for sync'); + } else if (bootContext.unreadable) { + log.warn('appStartupManager - Boot context unreadable: whether locations expired is decided app by app'); } else { const locationsExpired = (bootContext.cleanShutdown && bootContext.downtimeMs > SIGTERM_EXPIRY_MS) || bootContext.downtimeMs > RUNNING_EXPIRY_MS; @@ -337,6 +448,11 @@ async function manageAppsOnBoot(bootContext) { return; } + if (fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { + log.error('appStartupManager - Node is out of service for a host stall; its own removal unloads the apps'); + return; + } + if (fluxNetworkHelper.isNodeDos()) { log.error('appStartupManager - Node is in DOS state, removing all apps'); await removeAllApps('Node DOS'); diff --git a/ZelBack/src/services/appLifecycle/appUninstaller.js b/ZelBack/src/services/appLifecycle/appUninstaller.js index 53c0635724..785174bb9f 100644 --- a/ZelBack/src/services/appLifecycle/appUninstaller.js +++ b/ZelBack/src/services/appLifecycle/appUninstaller.js @@ -7,9 +7,10 @@ const messageHelper = require('../messageHelper'); const dockerService = require('../dockerService'); const dbHelper = require('../dbHelper'); const globalState = require('../utils/globalState'); +const folderStates = require('../appMonitoring/folderStates'); const log = require('../../lib/log'); const { - localAppsInformation, globalAppsInformation, globalAppsMessages, ANNOUNCE_CYCLE_WAIT_MS, + localAppsInformation, globalAppsInformation, globalAppsMessages, } = require('../utils/appConstants'); const config = require('config'); // const advancedWorkflows = require('./advancedWorkflows'); // Moved to dynamic require to avoid circular dependency @@ -21,10 +22,13 @@ const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const { stopAppMonitoring } = require('../appManagement/appInspector'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); +const appPlacements = require('../appManagement/appPlacements'); const volumeService = require('../utils/volumeService'); const fluxEventBus = require('../utils/fluxEventBus'); +const appSlice = require('./appSlice'); const { Privilege, authOf } = require('../utils/privileges'); const { RemovalOutcome } = require('../utils/removalOutcome'); +const { untilSucceeds, BOOT_STEP_RETRY_MS } = require('../utils/untilSucceeds'); /** * The node does not hold this application. @@ -57,6 +61,83 @@ function setOnComponentRemoved(callback) { onComponentRemoved = callback; } +// Apps whose removal stopped part way, each being run again until it finishes, with +// whether the run that finishes it is still to tell the network. +const unfinishedRemovals = new Map(); + +/** + * Runs the removal of an app that stopped part way again until it finishes. Until + * then the app is held as departing: the reconciler never starts it and the node's + * announcements leave it out. A call for an app already held does nothing. + * @param {string} appName + * @param {boolean} force Passed to each removal. + * @param {boolean} announce Whether the removal that finishes tells the network; it + * does not once the network has been told meanwhile (announceRemoval). + * @returns {void} + */ +function finishRemoval(appName, force, announce) { + if (unfinishedRemovals.has(appName)) return; + const pending = { announce }; + unfinishedRemovals.set(appName, pending); + globalState.departingApps.enter(appName); + log.warn(`${appName} was not fully removed; held stopped and unannounced until it is`); + // first run once the removal that stopped has released the node's removal lock + serviceHelper.delay(BOOT_STEP_RETRY_MS).then(() => untilSucceeds(`Removal of ${appName}`, async () => { + // Another install or removal holding the node is waited out, never run beside: + // a forced removal takes the removal lock without asking and clears it when it + // ends. Asked in the same turn as the removal starts, so nothing starts between. + const holder = globalState.operationHolding(); + if (holder) throw new Error(`another ${holder} holds the node`); + // eslint-disable-next-line no-use-before-define + const outcome = await removeAppLocally(appName, null, force, true, pending.announce); + if (outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + throw new Error(`${appName} was not removed (${outcome})`); + } + })).then(() => { + unfinishedRemovals.delete(appName); + globalState.departingApps.leave(appName); + }); +} + +/** + * Tells the network this node no longer holds the app (fluxappremoved), and drops it + * from the running apps cache. For an app whose removal from this node is complete. + * @param {string} appName + * @returns {Promise} + */ +async function announceRemoval(appName) { + // An announcement cycle that took its list before this removal marked the + // app still names it, and a claim arriving after this message re-creates the + // row it cleared. Waiting for the cycle in flight puts the two on the wire in + // the order they happened, so the claim is applied first and this clears it. + // A cycle starting from here on reads the mark and leaves the app out. A cycle + // is in flight only from its snapshot to its broadcast, and every call in that + // span has its own time limit. + await globalState.announceCycle.waitReady({ waitAll: false }); + const ip = await fluxNetworkHelper.getLocalSocketAddress(); + if (!ip) { + log.warn(`${appName} removed without announcing it - this node's own address is unknown, so peers hold its location row until it expires`); + return; + } + const appRemovedMessage = { + type: 'fluxappremoved', + version: 1, + appName, + ip, + broadcastedAt: Date.now(), + }; + log.info('Broadcasting appremoved message to the network'); + await fluxCommunicationMessagesSender.broadcastMessageToAll(appRemovedMessage); + // a removal still being finished has nothing left to tell + const pending = unfinishedRemovals.get(appName); + if (pending) pending.announce = false; + const { runningAppsCache } = globalState; + if (runningAppsCache.has(appName)) { + runningAppsCache.delete(appName); + log.info(`Removed ${appName} from running apps cache`); + } +} + /** * Stop Syncthing app and clean up cache * @param {string} monitoredName - Monitored app name @@ -77,7 +158,7 @@ async function stopSyncthingAndCleanup(monitoredName, appId, res) { // and one kept past the removal offers a volume this node no longer has - // outranking a node that still holds the app and seeding an empty folder in its // place. - globalState.folderHoldings?.delete(appId); + folderStates.removed(appId); globalState.seedMarks.delete(appId); try { @@ -844,6 +925,10 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // the same finally, and a refused duplicate must not release the lock the removal // actually running is holding. let acquired = false; + // Set once a teardown has begun: a failure from there on leaves the app part + // removed, and an app missing a component cannot run, so it is the whole app. + let tornDownApp = null; + let announced = false; try { // Normalise to the bare identifier this function reasons about: a caller may // pass the flux-prefixed docker name (e.g. the syncthing flow), which would @@ -920,21 +1005,15 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // get it from locally available Specifications const allApps = await availableApps(); appSpecifications = allApps.find((a) => a.name === appName); - // get it from permanent messages + // get it from permanent messages: the app's newest, read through the + // index on name, height and timestamp if (!appSpecifications) { - const query = {}; - const projection = { projection: { _id: 0 } }; - const messages = await dbHelper.findInDatabase(database, globalAppsMessages, query, projection); - const appMessages = messages.filter((message) => { - const specifications = message.appSpecifications; - return specifications.name === appName; - }); - let currentSpecifications; - appMessages.forEach((message) => { - if (!currentSpecifications || message.height > currentSpecifications.height) { - currentSpecifications = message; - } - }); + const [currentSpecifications] = await dbHelper.findInDatabase( + database, + globalAppsMessages, + { 'appSpecifications.name': appName }, + { projection: { _id: 0 }, sort: { height: -1, timestamp: -1 }, limit: 1 }, + ); if (currentSpecifications && currentSpecifications.height) { ({ appSpecifications } = currentSpecifications); } @@ -952,6 +1031,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen appSpecifications = await checkAndDecryptAppSpecs(appSpecifications); appSpecifications = specificationFormatter(appSpecifications); + tornDownApp = appName; if (appSpecifications.version >= 4 && !isComponent) { // it is a composed application // eslint-disable-next-line no-restricted-syntax @@ -968,6 +1048,7 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen } else { await hardUninstallApplication(appName, appId, appSpecifications, res, stopAppMonitoring, force); } + if (!isComponent) await appSlice.release(appName); // clear node-local runtime state (operator stop lock, crash backoff) for the // removed component(s) so a later reinstall starts from a clean slate @@ -986,38 +1067,9 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (onComponentRemoved) onComponentRemoved(identifier); } - fluxEventBus.publish('app:removed', { name: appName }); - if (sendMessage) { - // An announcement cycle that took its list before this removal marked the - // app still names it, and a claim arriving after this message re-creates the - // row it cleared. Waiting for the cycle in flight puts the two on the wire in - // the order they happened, so the claim is applied first and this clears it. - // A cycle starting from here on reads the mark and leaves the app out. - // Bounded: a wedged cycle must not hold up the node's removals. - await globalState.announceCycle.readyTimeout(ANNOUNCE_CYCLE_WAIT_MS); - const ip = await fluxNetworkHelper.getLocalSocketAddress(); - if (ip) { - const broadcastedAt = Date.now(); - const appRemovedMessage = { - type: 'fluxappremoved', - version: 1, - appName, - ip, - broadcastedAt, - }; - log.info('Broadcasting appremoved message to the network'); - // broadcast messages about app removed to all peers - await fluxCommunicationMessagesSender.broadcastMessageToAll(appRemovedMessage); - // Remove app from running apps cache - const { runningAppsCache } = globalState; - if (runningAppsCache.has(appName)) { - runningAppsCache.delete(appName); - log.info(`Removed ${appName} from running apps cache`); - } - } else { - log.warn(`${appName} removed without announcing it - this node's own address is unknown, so peers hold its location row until it expires`); - } + await announceRemoval(appName); + announced = true; } if (!isComponent) { @@ -1098,6 +1150,14 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } await dbHelper.findOneAndDeleteInDatabase(appsDatabase, localAppsInformation, appsQuery, appsProjection); + // The app's volume is gone, and the record of when this node took it with it. + // A record left behind is never read, as the app is not installed, and boot + // removes it, so the removal stands whether this delete succeeds or not. + try { + await appPlacements.removePlacement(appName); + } catch (error) { + log.warn(`removeAppLocally - the placement record of ${appName} was not removed: ${error.message}`); + } // The app is gone for good: nothing reconciles an app with no row, so its // removal records have no reader left. Only this full-uninstall path clears // them - softRemoveAppLocally deletes the row too, but as one step of a @@ -1105,6 +1165,9 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen // exactly what stops a teardown that fails part way being read as // tampering. dockerService.clearFluxRemovedContainers(appName); + // Nor is there an app left for boot's location check to decide: an install + // of the same name from here on is a new app. + globalState.unconfirmedApps.delete(appName); const databaseStatus2 = { status: 'Database cleaned', }; @@ -1114,6 +1177,9 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen if (res.flush) res.flush(); } } + // Published once the removal has finished, the app's row included. + fluxEventBus.publish('app:removed', { name: appName }); + const appRemovalResponseDone = messageHelper.createSuccessMessage(`Removal step done. Result: Flux App ${appName} was successfuly removed`); log.info(appRemovalResponseDone); @@ -1141,12 +1207,12 @@ async function removeAppLocally(app, res, force = false, endResponse = true, sen } } // A specification that cannot be found anywhere means the node is not holding - // this app, which answers a caller asking for it to be gone. Every other error - // leaves it possibly here, whole or in part, and only the caller can decide what - // that costs it. - return error instanceof AppNotFoundError - ? RemovalOutcome.NOT_INSTALLED - : RemovalOutcome.FAILED; + // this app, which answers a caller asking for it to be gone. An app whose + // teardown had begun, of the whole app or of one component, is removed whole in + // the background, and the network is told once it is gone. Any other error leaves the app as it was. + if (error instanceof AppNotFoundError) return RemovalOutcome.NOT_INSTALLED; + if (tornDownApp) finishRemoval(tornDownApp, force, !announced); + return RemovalOutcome.FAILED; } finally { if (acquired) globalState.removalInProgress = false; if (departingName) { @@ -1339,6 +1405,8 @@ async function removeAppLocallyApi(req, res) { } module.exports = { + announceRemoval, + finishRemoval, hardUninstallComponent, hardUninstallApplication, softUninstallComponent, diff --git a/ZelBack/src/services/appManagement/appController.js b/ZelBack/src/services/appManagement/appController.js index b32bba5be1..44e3cfefce 100644 --- a/ZelBack/src/services/appManagement/appController.js +++ b/ZelBack/src/services/appManagement/appController.js @@ -195,11 +195,11 @@ async function operatorTargetIds(appname) { if (!ids.length) { throw new Error(`Application ${appName} was not changed: this node cannot determine its components`); } - // The component whose start the election decides, read from the spec as the - // election reads it, so the two agree on which one that is. + // The components whose start the election decides, read from the spec as the + // election reads it, so the two agree on which they are. const { inPlace: [spec] } = await appQueryService.decryptEnterpriseApps([installedRes.data[0]]); - const electedId = mountParser.electedComponentIdentifier(spec); - if (!appname.includes('_')) return { ids, appName, electedId }; + const electedIds = mountParser.gComponentIdentifiers(spec); + if (!appname.includes('_')) return { ids, appName, electedIds }; // A component is addressed by name, and a name that is not one of this app's // components addresses nothing. Taking it verbatim wrote a durable operator // lock under a component that does not exist - nothing clears one, and it holds @@ -207,11 +207,27 @@ async function operatorTargetIds(appname) { if (!ids.includes(appname)) { throw new Error(`Component ${appname} is not installed on this node`); } - return { ids: [appname], appName: appname, electedId }; + return { ids: [appname], appName: appname, electedIds }; +} + +/** + * Whether this node holds the app whose g: components these are: it runs one, + * is becoming the primary of one, or has asked the reconciler to run one. Read + * for the app and not the component, because the election decides the app - a + * component stopped on the node that runs the others was stopped on its primary. + * @param {string[]} electedIds The app's g: component identifiers. + * @returns {Promise} + */ +async function holdsElectedApp(electedIds) { + const committed = appReconciler.committedIdentifiers(); + const promoting = primaryRoleChanges.promotingIdentifiers(); + if (electedIds.some((id) => committed.includes(id) || promoting.includes(id))) return true; + const actual = await Promise.all(electedIds.map((id) => appReconciler.dockerActual(id))); + return actual.some((each) => each.running === true); } async function setAppOperatorStopped(appname, stopped, { awaitPass = false, force = false, alsoRestart = false } = {}) { - const { ids, appName, electedId } = await operatorTargetIds(appname); + const { ids, appName, electedIds } = await operatorTargetIds(appname); // Components come up in compose order and go down in the reverse of it, so a // dependency outlives what writes to it: the database stops after the server it // serves, not before it. awaitPass holds each component's pass open before the @@ -232,20 +248,16 @@ async function setAppOperatorStopped(appname, stopped, { awaitPass = false, forc let awaitsElection = false; // eslint-disable-next-line no-await-in-loop const actuated = await appReconciler.applyIntent(id, async () => { - // A start of the component the election decides, while it is locked, + // A start of a component the election decides, while it is locked, // leaves the lock for the election to lift: lifted here, the component // would be held by nothing until the election next ran, and a standby // could take it in between. See appsRuntimeState.requestOperatorStart. - awaitsElection = !stopped && id === electedId && await appsRuntimeState.requestOperatorStart(id); + const elected = electedIds.includes(id); + awaitsElection = !stopped && elected && await appsRuntimeState.requestOperatorStart(id); // Read inside the slot, so no pass moves the container between this read // and the lock. A container docker cannot read is not one known to run. - const lockOptions = stopped && id === electedId - ? { - force, - asPrimary: appReconciler.committedIdentifiers().includes(id) - || primaryRoleChanges.promotingIdentifiers().includes(id) - || (await appReconciler.dockerActual(id)).running === true, - } + const lockOptions = stopped && elected + ? { force, asPrimary: await holdsElectedApp(electedIds) } : { force }; if (!awaitsElection) await appsRuntimeState.setOperatorStopped(id, stopped, lockOptions); // Raised inside the same slot as the lock, so a pass cannot read one @@ -380,7 +392,7 @@ async function containersRestarted(ids) { // eslint-disable-next-line no-restricted-syntax for (const id of ids) { // eslint-disable-next-line no-await-in-loop - const state = await appsRuntimeState.getState(id); + const state = await appsRuntimeState.getState(id).catch(() => null); if (!state) return { settled: false, reason: 'its state could not be read' }; if ((state.actuatedRestartGeneration ?? 0) < (state.restartGeneration ?? 0)) { // eslint-disable-next-line no-await-in-loop diff --git a/ZelBack/src/services/appManagement/appInspector.js b/ZelBack/src/services/appManagement/appInspector.js index 4de7b493d6..f08ec65b21 100644 --- a/ZelBack/src/services/appManagement/appInspector.js +++ b/ZelBack/src/services/appManagement/appInspector.js @@ -7,6 +7,7 @@ const logCursor = require('../utils/logCursor'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const globalState = require('../utils/globalState'); const cpuBurstHelper = require('../utils/cpuBurstHelper'); +const cpuThrottleExemption = require('../utils/cpuThrottleExemption'); const log = require('../../lib/log'); const { getContainerStorage } = require('../utils/appUtilities'); const { Privilege, authOf } = require('../utils/privileges'); @@ -317,12 +318,9 @@ function extractSample(stats) { // space. `docker stats` subtracts it, so every consumer that reports a // container's memory subtracts it too, and without this figure none of them // can: they subtract zero and report a number that climbs with disk reads. - // cgroup v2 names it inactive_file and v1 names it cache; they are not the - // same counter (v1's is the whole page cache, v2's the reclaimable part) but - // docker's own CLI reads them from the same slot, so this does too. Null - // where neither is present, which leaves the consumer exactly where it was. - memoryCache: stats.memory_stats?.stats?.inactive_file - ?? stats.memory_stats?.stats?.cache ?? null, + // cgroup v2 names it inactive_file. Null where it is absent, which leaves + // the consumer exactly where it was. + memoryCache: stats.memory_stats?.stats?.inactive_file ?? null, ioRead: sumBlkio('read'), ioWrite: sumBlkio('write'), // EVERY interface, not just eth0. appNetworkLinker connects a container to @@ -361,9 +359,6 @@ function expandSample(sample) { cpu_usage: { total_usage: sample.cpuTotalBefore }, system_cpu_usage: sample.cpuSystemBefore, }, - // Reported under the cgroup v2 name whichever key it was read from: a - // consumer's `inactive_file ?? cache` chain then settles on the first, and - // the value is the same either way. memory_stats: { usage: sample.memoryUsage, limit: sample.memoryLimit, @@ -908,6 +903,35 @@ function sampleCpuLoad(sample, specifiedCpu) { return ((cpuUsage / systemCpuUsage) * sample.onlineCpus * 100) / specifiedCpu || 0; } +/** + * Hold a throttle-exempt container at its full specified cpu instead of deciding on it. + * + * Restoring is not optional. A game server lowered before it became exempt (or before this + * node read the policy) would otherwise stay at 80% forever: the decision below measures load + * against the reduced allocation, so a saturated server never looks idle enough to be raised. + * @param {string} containerName - The app or component container name + * @param {number} specifiedCpu - The cpu the app or component specified + * @param {object} inspect - The container's docker inspect + * @param {object} monitored - The monitoring entry, which may have gone mid-pass + * @param {number} decisionAt - The watermark to stamp + * @returns {Promise} + */ +async function holdExemptAtSpecifiedCpu(containerName, specifiedCpu, inspect, monitored, decisionAt) { + const nanoCpus = inspect.HostConfig?.NanoCpus; + const specifiedNanoCpus = Math.round(specifiedCpu * 1e9); + if (nanoCpus && nanoCpus < specifiedNanoCpus) { + await dockerService.appDockerUpdateCpu(containerName, specifiedNanoCpus); + log.info(`checkApplicationsCpuUSage ${containerName} throttle-exempt, restoring cpu 100.`); + } else { + log.info(`checkApplicationsCpuUSage ${containerName} throttle-exempt, skipping CPU throttling`); + } + // Guarded for the same reason as the watermark writes in the caller. + if (monitored) { + // eslint-disable-next-line no-param-reassign + monitored.lastCpuDecisionAt = decisionAt; + } +} + /** * Check if applications are throttling CPU and adjust CPU limits * @param {object} appsMonitored - Applications monitoring data @@ -951,6 +975,13 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { // eslint-disable-next-line no-continue continue; } + // Game servers deployed by the games hub are never throttled. See cpuThrottleExemption. + if (inspect && cpuThrottleExemption.isCpuThrottleExempt(app.name)) { + // eslint-disable-next-line no-await-in-loop + await holdExemptAtSpecifiedCpu(app.name, app.cpu, inspect, appsMonitored[app.name], decisionAt); + // eslint-disable-next-line no-continue + continue; + } if (inspect && stats && stats.length > 4) { const nanoCpus = inspect.HostConfig.NanoCpus; let cpuThrottlingRuns = 0; @@ -1031,6 +1062,13 @@ async function checkApplicationsCpuUSage(appsMonitored, installedApps) { // eslint-disable-next-line no-continue continue; } + // Exemption is by APP name: every component of a hub game server is held at spec. + if (inspect && cpuThrottleExemption.isCpuThrottleExempt(app.name)) { + // eslint-disable-next-line no-await-in-loop + await holdExemptAtSpecifiedCpu(compName, appComponent.cpu, inspect, appsMonitored[compName], decisionAt); + // eslint-disable-next-line no-continue + continue; + } if (inspect && stats && stats.length > 4) { const nanoCpus = inspect.HostConfig.NanoCpus; let cpuThrottlingRuns = 0; diff --git a/ZelBack/src/services/appManagement/appPlacements.js b/ZelBack/src/services/appManagement/appPlacements.js new file mode 100644 index 0000000000..774ee7afb9 --- /dev/null +++ b/ZelBack/src/services/appManagement/appPlacements.js @@ -0,0 +1,167 @@ +const config = require('config'); +const log = require('../../lib/log'); +const dbHelper = require('../dbHelper'); +const fluxEventBus = require('../utils/fluxEventBus'); +const { AsyncGate } = require('../utils/asyncGate'); + +// When this node took each app it holds: one record per app, keyed by name. +// +// The record follows the app's data on this node. It is written when a hard +// install creates the app's volume, and deleted when a removal deletes the +// volume; a soft redeploy keeps the volume and leaves the record as it is. So a +// hard redeploy, which starts from an empty volume, starts a new record. +// +// The announcer reads runningSince from here and nowhere else, because the +// network's copy of this node's own location rows lapses in normal running - +// expiry, eviction, sigterm - while the app goes on running. +// +// A record outlives its app when a soft redeploy removes the app and its reinstall +// does not happen, which leaves the volume on disk and the record with it, when +// the app is removed while the backfill below is writing, or when the removal +// cannot delete it. Boot removes records of apps not installed. Only an installed app's record is read, and the app returns +// only through a hard install, which overwrites it. +// +// Kept apart from the app's specification in appsInformation: a spec update +// deletes and re-inserts that, and this record must outlive it. + +const appsLocalDatabase = config.database.appslocal.database; +const { appsPlacements } = config.database.appslocal.collections; +const { appsLocations } = config.database.appsglobal.collections; + +// Open once every app installed at start has a record. +const backfilledGate = new AsyncGate(); + +function localDatabase() { + return dbHelper.databaseConnection().db(appsLocalDatabase); +} + +/** + * One record per app name. + * @returns {Promise} + */ +async function prepareCollection() { + await dbHelper.ensureIndexes(localDatabase().collection(appsPlacements), [ + { key: { name: 1 }, unique: true, name: 'name_unique', recover: dbHelper.dedupeByKey }, + ]); +} + +/** + * Record that this node took an app now, over any record it had of the app: a + * hard install has just created the app's volume. + * @param {string} name App name. + * @returns {Promise} + */ +async function recordPlacement(name) { + const placedAt = new Date(); + await dbHelper.updateOneInDatabase( + localDatabase(), + appsPlacements, + { name }, + { $set: { name, placedAt } }, + { upsert: true }, + ); + fluxEventBus.publish('app:placementRecorded', { name, placedAt: placedAt.toISOString(), source: 'install' }); +} + +/** + * When this node took each named app, for the apps it has a record of. + * @param {Array} names App names. + * @returns {Promise>} placedAt by name. + */ +async function placementsOf(names) { + const records = await dbHelper.findInDatabase( + localDatabase(), + appsPlacements, + { name: { $in: names } }, + { projection: { _id: 0, name: 1, placedAt: 1 } }, + ); + return new Map(records.map(({ name, placedAt }) => [name, placedAt])); +} + +/** + * Forget an app whose volume this node has deleted. + * @param {string} name App name. + * @returns {Promise} + */ +async function removePlacement(name) { + const result = await dbHelper.removeDocumentsFromCollection(localDatabase(), appsPlacements, { name }); + if (result?.deletedCount) fluxEventBus.publish('app:placementRemoved', { name }); +} + +/** + * Remove the records of apps not installed here. + * @param {Array} installedNames Names of the apps installed here. + * @returns {Promise} How many were removed. + */ +async function removeStrayPlacements(installedNames) { + const removal = await dbHelper.removeDocumentsFromCollection(localDatabase(), appsPlacements, { name: { $nin: installedNames } }); + const removed = removal?.deletedCount ?? 0; + if (removed) log.info(`appPlacements - removed ${removed} record(s) of apps not installed here`); + fluxEventBus.publish('app:placementsTidied', { removed }); + return removed; +} + +/** + * Give each installed app without a record one. It is dated at the runningSince + * this node's own location row holds, so the app keeps the place among its + * instances the network already gives it, and dated now when there is no row. A + * record that exists is left as it is. Opens waitForBackfill. + * @param {Array} installedNames Names of the apps installed here. + * @param {string} localSocketAddr This node's address, which its rows are filed under. + * @returns {Promise<{created: number, fromLocationRows: number}>} What was written. + */ +async function backfillPlacements(installedNames, localSocketAddr) { + const recorded = await placementsOf(installedNames); + const missing = installedNames.filter((name) => !recorded.has(name)); + + const ownRows = missing.length + ? await dbHelper.findInDatabase( + dbHelper.databaseConnection().db(config.database.appsglobal.database), + appsLocations, + { ip: localSocketAddr, name: { $in: missing } }, + { projection: { _id: 0, name: 1, runningSince: 1 } }, + ) + : []; + const rowSince = new Map(ownRows.filter((row) => row.runningSince).map((row) => [row.name, new Date(row.runningSince)])); + const now = new Date(); + // eslint-disable-next-line no-restricted-syntax + for (const name of missing) { + const placedAt = rowSince.get(name) ?? now; + // eslint-disable-next-line no-await-in-loop + const result = await dbHelper.updateOneInDatabase( + localDatabase(), + appsPlacements, + { name }, + { $setOnInsert: { name, placedAt } }, + { upsert: true }, + ); + if (result?.upsertedCount) { + fluxEventBus.publish('app:placementRecorded', { name, placedAt: placedAt.toISOString(), source: rowSince.has(name) ? 'locationRow' : 'now' }); + } + } + + if (missing.length) { + log.info(`appPlacements - recorded ${missing.length} placement(s), ${rowSince.size} from this node's location rows`); + } + fluxEventBus.publish('app:placementsBackfilled', { created: missing.length, fromLocationRows: rowSince.size }); + backfilledGate.open(); + return { created: missing.length, fromLocationRows: rowSince.size }; +} + +/** + * Resolves once every app installed at start has a record. + * @returns {Promise} + */ +function waitForBackfill() { + return backfilledGate.wait(); +} + +module.exports = { + prepareCollection, + recordPlacement, + placementsOf, + removePlacement, + removeStrayPlacements, + backfillPlacements, + waitForBackfill, +}; diff --git a/ZelBack/src/services/appManagement/appsRuntimeState.js b/ZelBack/src/services/appManagement/appsRuntimeState.js index cf2ee0c389..717e693611 100644 --- a/ZelBack/src/services/appManagement/appsRuntimeState.js +++ b/ZelBack/src/services/appManagement/appsRuntimeState.js @@ -50,20 +50,17 @@ function canonical(identifier) { /** * Returns the persisted runtime-state document for a component identifier - * (e.g. `component_app`, or the app name for v1-3 apps), or null. + * (e.g. `component_app`, or the app name for v1-3 apps), or null when it has + * none. Rejects when the read fails: an unread record is not an absent one, and + * every lock and ladder this document holds would read as clear. * * @param {string} identifier * @returns {Promise} */ async function getState(rawIdentifier) { const identifier = canonical(rawIdentifier); - try { - const database = collection(); - return await dbHelper.findOneInDatabase(database, appsRuntimeState, { identifier }, { projection: { _id: 0 } }); - } catch (err) { - log.error(`appsRuntimeState - failed to read state for ${identifier}: ${err.message}`); - return null; - } + const database = collection(); + return dbHelper.findOneInDatabase(database, appsRuntimeState, { identifier }, { projection: { _id: 0 } }); } function isDuplicateKeyError(err) { @@ -149,9 +146,9 @@ async function setOperatorStopped(identifier, stopped, opts = {}) { await setFields(identifier, fields); } -// The stop lock and its start request, read without getState's swallow: these -// readers decide whether to lift the lock, and a read failure answered as "no -// record" is a lock lifted on no evidence. +// The stop lock and its start request, the only fields these readers need. +// Rejects when the read fails: a lock lifted on an unread record is lifted on no +// evidence. function readLock(identifier) { return dbHelper.findOneInDatabase( collection(), @@ -162,9 +159,8 @@ function readLock(identifier) { } /** - * Whether the operator stop lock is on, read without getState's swallow: for a - * caller about to commit a component to running, an unread lock is not an - * absent one. + * Whether the operator stop lock is on: for a caller about to commit a component + * to running, an unread lock is not an absent one. * * @param {string} rawIdentifier * @returns {Promise} @@ -298,26 +294,9 @@ async function recordRestartGeneration(identifier, generation) { } /** - * Whether the operator has deliberately stopped this component, so the - * reconciler (and the masterSlave/syncthing deciders) must leave it stopped. - * - * @param {string} identifier - * @returns {Promise} - */ -async function isOperatorStopped(identifier) { - const state = await getState(identifier); - return state?.operatorStopped === true; -} - -/** - * The operator's stop lock and the mode they asked for, from ONE read. - * - * They are two fields of one document, and a caller needing both must not ask - * twice: getState returns null for a read failure exactly as it does for "no - * record", so a second read that failed reported no force flag and turned the - * operator's "kill now" into a drain they did not ask for. Answering both from a - * single read is a window that cannot open - and one round-trip fewer on every - * stop the reconciler performs. + * The operator's stop lock and the mode they asked for, from ONE read: two + * fields of one document, answered together so they always describe the same + * moment. Rejects when the read fails. * * `startRequested` is a start the operator has asked for and the election has * not yet decided; the lock stands until it has. `asPrimary` is whether this @@ -638,18 +617,15 @@ async function removeControllerState(rawIdentifier) { } /** - * Drops all runtime state for a component (on uninstall). + * Drops all runtime state for a component (on uninstall). Rejects when the + * delete fails: a stop lock left behind holds a later reinstall stopped. * * @param {string} identifier */ async function remove(rawIdentifier) { const identifier = canonical(rawIdentifier); - try { - const database = collection(); - await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier }); - } catch (err) { - log.error(`appsRuntimeState - failed to remove state for ${identifier}: ${err.message}`); - } + const database = collection(); + await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier }); } /** @@ -706,80 +682,81 @@ async function getVolumeImage(rawIdentifier) { * the backoff history on the other - so dedupe must merge field-wise rather * than keep one doc whole: dropping a doc could drop a real operator lock, * whose loss would auto-start a deliberately stopped app. + * + * Rejects when any step fails, so boot's schema preparation runs it again. */ async function prepareCollection() { - try { - const database = collection(); - const all = await dbHelper.findInDatabase(database, appsRuntimeState, {}, { projection: { _id: 0 } }); - const byIdentifier = new Map(); - // eslint-disable-next-line no-restricted-syntax - for (const doc of all) { - const list = byIdentifier.get(doc.identifier) || []; - list.push(doc); - byIdentifier.set(doc.identifier, list); - } - // eslint-disable-next-line no-restricted-syntax - for (const [identifier, twins] of byIdentifier) { - if (twins.length > 1) { - // Oldest first, so every field any twin carries survives and the newest - // value of each wins. Fields scatter across twins - that is the whole - // reason this merge exists - so taking them from one document would - // drop exactly what the scatter put on the other, and a field added to - // this document later would be lost without ever being named here. - // What follows overrides the ones whose correct value is not simply - // "whichever was written last". - const oldestFirst = [...twins].sort((a, b) => (a.updatedAt || 0) - (b.updatedAt || 0)); - // The image record is one claim - that the file at this path carries - // this UUID - so it moves as a pair. Merged field-wise it could take - // the path from one twin and the stamp from another, and a record - // saying an image carries a UUID it never carried refuses that volume - // for good: the refusal returns before the line that would replace it. - const withImage = oldestFirst.filter((t) => t.volumeImagePath); - const image = withImage.length ? withImage[withImage.length - 1] : null; - const merged = { - ...Object.assign({}, ...oldestFirst), - ...(image ? { volumeImagePath: image.volumeImagePath, volumeFsUuid: image.volumeFsUuid || null } : {}), - identifier, - // a lock anywhere is a lock: never auto-start a deliberately stopped app - operatorStopped: twins.some((t) => t.operatorStopped === true), - // a force anywhere is a force: a kill must never be merged down into a - // graceful stop the operator did not ask for - operatorStopForce: twins.some((t) => t.operatorStopForce === true), - // a start waits on the election only if no locked twin withholds it: a - // stop without one is the operator's later word - operatorStartRequested: twins.some((t) => t.operatorStopped === true) - && twins.every((t) => t.operatorStopped !== true || t.operatorStartRequested === true), - // the highest request wins and the lowest actuation does, so a restart - // asked for on either doc still bounces the container once - restartGeneration: Math.max(0, ...twins.map((t) => t.restartGeneration || 0)), - actuatedRestartGeneration: Math.min(...twins.map((t) => t.actuatedRestartGeneration || 0)), - // likewise, a heal-removal anywhere means a heal may be mid-flight: keep - // the reconciler on the recreate path rather than the uninstall path - networkHealRemoval: twins.some((t) => t.networkHealRemoval === true), - networkHealHistory: [...new Set(twins.flatMap((t) => t.networkHealHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), - restartHistory: [...new Set(twins.flatMap((t) => t.restartHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), - autoRestartWindow: [...new Set(twins.flatMap((t) => t.autoRestartWindow || []))].sort((a, b) => a - b).slice(-RESTART_BURST_COUNT), - updatedAt: Math.max(...twins.map((t) => t.updatedAt || 0)), - }; - // the read projects _id away, but a spread of whatever the documents - // carry must not be the thing that reintroduces it into a $set - delete merged._id; - const newestExit = twins.filter((t) => t.lastDiedAt !== undefined).sort((a, b) => b.lastDiedAt - a.lastDiedAt)[0]; - if (newestExit) { - merged.lastExitCode = newestExit.lastExitCode; - merged.lastDiedAt = newestExit.lastDiedAt; - } - log.warn(`appsRuntimeState - merged ${twins.length} duplicate docs for ${identifier} (operatorStopped=${merged.operatorStopped}, ${merged.restartHistory.length} restart entries)`); - // eslint-disable-next-line no-await-in-loop - await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier }); - // eslint-disable-next-line no-await-in-loop - await dbHelper.updateOneInDatabase(database, appsRuntimeState, { identifier }, { $set: merged }, { upsert: true }); + const database = collection(); + const all = await dbHelper.findInDatabase(database, appsRuntimeState, {}, {}); + const byIdentifier = new Map(); + // eslint-disable-next-line no-restricted-syntax + for (const doc of all) { + const list = byIdentifier.get(doc.identifier) || []; + list.push(doc); + byIdentifier.set(doc.identifier, list); + } + // eslint-disable-next-line no-restricted-syntax + for (const [identifier, twins] of byIdentifier) { + if (twins.length > 1) { + // Oldest first, so every field any twin carries survives and the newest + // value of each wins. Fields scatter across twins - that is the whole + // reason this merge exists - so taking them from one document would + // drop exactly what the scatter put on the other, and a field added to + // this document later would be lost without ever being named here. + // What follows overrides the ones whose correct value is not simply + // "whichever was written last". + const oldestFirst = [...twins].sort((a, b) => (a.updatedAt || 0) - (b.updatedAt || 0)); + // The image record is one claim - that the file at this path carries + // this UUID - so it moves as a pair. Merged field-wise it could take + // the path from one twin and the stamp from another, and a record + // saying an image carries a UUID it never carried refuses that volume + // for good: the refusal returns before the line that would replace it. + const withImage = oldestFirst.filter((t) => t.volumeImagePath); + const image = withImage.length ? withImage[withImage.length - 1] : null; + const merged = { + ...Object.assign({}, ...oldestFirst), + ...(image ? { volumeImagePath: image.volumeImagePath, volumeFsUuid: image.volumeFsUuid || null } : {}), + identifier, + // a lock anywhere is a lock: never auto-start a deliberately stopped app + operatorStopped: twins.some((t) => t.operatorStopped === true), + // a force anywhere is a force: a kill must never be merged down into a + // graceful stop the operator did not ask for + operatorStopForce: twins.some((t) => t.operatorStopForce === true), + // a start waits on the election only if no locked twin withholds it: a + // stop without one is the operator's later word + operatorStartRequested: twins.some((t) => t.operatorStopped === true) + && twins.every((t) => t.operatorStopped !== true || t.operatorStartRequested === true), + // the highest request wins and the lowest actuation does, so a restart + // asked for on either doc still bounces the container once + restartGeneration: Math.max(0, ...twins.map((t) => t.restartGeneration || 0)), + actuatedRestartGeneration: Math.min(...twins.map((t) => t.actuatedRestartGeneration || 0)), + // likewise, a heal-removal anywhere means a heal may be mid-flight: keep + // the reconciler on the recreate path rather than the uninstall path + networkHealRemoval: twins.some((t) => t.networkHealRemoval === true), + networkHealHistory: [...new Set(twins.flatMap((t) => t.networkHealHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), + restartHistory: [...new Set(twins.flatMap((t) => t.restartHistory || []))].sort((a, b) => a - b).slice(-MAX_HISTORY), + autoRestartWindow: [...new Set(twins.flatMap((t) => t.autoRestartWindow || []))].sort((a, b) => a - b).slice(-RESTART_BURST_COUNT), + updatedAt: Math.max(...twins.map((t) => t.updatedAt || 0)), + }; + // the read projects _id away, but a spread of whatever the documents + // carry must not be the thing that reintroduces it into a $set + delete merged._id; + const newestExit = twins.filter((t) => t.lastDiedAt !== undefined).sort((a, b) => b.lastDiedAt - a.lastDiedAt)[0]; + if (newestExit) { + merged.lastExitCode = newestExit.lastExitCode; + merged.lastDiedAt = newestExit.lastDiedAt; } + log.warn(`appsRuntimeState - merged ${twins.length} duplicate docs for ${identifier} (operatorStopped=${merged.operatorStopped}, ${merged.restartHistory.length} restart entries)`); + // The merge lands on one twin before the others go, so a failure part way + // leaves duplicates for the next start to merge, never no record at all. + const kept = oldestFirst[oldestFirst.length - 1]._id; + // eslint-disable-next-line no-await-in-loop + await dbHelper.updateOneInDatabase(database, appsRuntimeState, { _id: kept }, { $set: merged }); + // eslint-disable-next-line no-await-in-loop + await dbHelper.removeDocumentsFromCollection(database, appsRuntimeState, { identifier, _id: { $ne: kept } }); } - await database.collection(appsRuntimeState).createIndex({ identifier: 1 }, { unique: true, name: 'identifier_unique' }); - } catch (err) { - log.error(`appsRuntimeState - failed to prepare collection: ${err.message}`); } + await database.collection(appsRuntimeState).createIndex({ identifier: 1 }, { unique: true, name: 'identifier_unique' }); } module.exports = { @@ -789,7 +766,6 @@ module.exports = { requestOperatorStart, releaseOperatorStart, operatorStoppedOrThrow, - isOperatorStopped, operatorStopState, operatorHeldIdentifiers, recordRestart, diff --git a/ZelBack/src/services/appMessaging/appHashSyncService.js b/ZelBack/src/services/appMessaging/appHashSyncService.js index 1fda027fc4..9a727db4df 100644 --- a/ZelBack/src/services/appMessaging/appHashSyncService.js +++ b/ZelBack/src/services/appMessaging/appHashSyncService.js @@ -9,6 +9,7 @@ const messageVerifier = require('./messageVerifier'); const appValidator = require('../appRequirements/appValidator'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const { serialiseAndSignFluxBroadcast } = require('../utils/fluxBroadcastHelper'); const { peerManager } = require('../utils/peerState'); @@ -21,6 +22,7 @@ const { appSyncEvents, EVENTS } = require('../utils/appSyncEvents'); const { HASH_EXPIRY_BLOCKS, HASH_RETRY_BACKOFF } = require('../utils/appConstants'); const log = require('../../lib/log'); const fluxEventBus = require('../utils/fluxEventBus'); +const globalState = require('../utils/globalState'); const { isBefore, messagesThatCount } = require('../utils/appMessageChain'); const { invalidMessages } = require('../invalidMessages'); const { Privilege, authOf } = require('../utils/privileges'); @@ -373,6 +375,7 @@ async function processMessages(messages, records, onProgress) { if (appSpecFormatted.version >= 8 && appSpecFormatted.enterprise) { try { + verifyAppNameAndOwner(appSpecFormatted.name, appSpecFormatted.owner); const decrypted = await checkAndDecryptAppSpecs( appSpecFormatted, { daemonHeight: height, owner: appSpecFormatted.owner }, @@ -413,6 +416,7 @@ async function processMessages(messages, records, onProgress) { let prevSpecsForVerification = prevSpecs; if (prevSpecs.version >= 8 && prevSpecs.enterprise) { try { + verifyAppNameAndOwner(prevSpecs.name, prevSpecs.owner); const decrypted = await checkAndDecryptAppSpecs( prevSpecs, { daemonHeight: prevMsg.height, owner: prevSpecs.owner }, ); @@ -465,8 +469,9 @@ async function processMessages(messages, records, onProgress) { } // 5. Batch insert permanent messages (ordered so insertedCount is deterministic on failure) + let inserted = 0; if (permInserts.length > 0) { - let inserted = permInserts.length; + inserted = permInserts.length; await appsGlobalDb.collection(globalAppsMessages).insertMany(permInserts, { ordered: true }) .catch((err) => { inserted = err.result?.insertedCount ?? 0; @@ -476,6 +481,21 @@ async function processMessages(messages, records, onProgress) { processed += inserted; } + // The registry rows of the apps whose messages were stored, before their hashes are + // marked. A registry not built yet is built in full from every stored message before + // the node is ready (dbReady), so only a built one is kept current here. Rows not + // written leave the hashes unmarked, and the next retry finds the messages stored and + // writes the rows (alignStoredMessagesWithPayments). + if (inserted > 0 && globalState.dbReady) { + const apps = [...new Set(permInserts.slice(0, inserted).map((message) => message.appSpecifications.name))]; + try { + await messageVerifier.storeGoverningAppSpecifications(apps); + } catch (err) { + log.error(`processMessages registry rows: ${err.message}`); + hashMarkOps = []; + } + } + // 6. Batch mark hashes if (hashMarkOps.length > 0) { await daemonDb.collection(appsHashesCollection).bulkWrite(hashMarkOps, { ordered: false }) @@ -773,10 +793,22 @@ async function syncMissingHashes(options = {}) { // Update backoff for unresolved hashes const finalMissing = await getMissingHashes({ force: false, currentHeight }); + // A hash whose message this node already stores waits on no peer: the next pass + // writes its registry row from the stored copy, so it is neither backed off nor + // given up on. + const stored = new Set(); + for (let i = 0; i < finalMissing.length; i += CHUNK_SIZE) { + const hashValues = finalMissing.slice(i, i + CHUNK_SIZE).map((h) => h.hash); + // eslint-disable-next-line no-await-in-loop + const found = await dbHelper.findInDatabase(appsGlobalDb, globalAppsMessages, { hash: { $in: hashValues } }, { projection: { _id: 0, hash: 1 } }); + found.forEach((m) => stored.add(m.hash)); + } + let unreachable = 0; let minNextRetry = null; const backoffOps = []; for (const hashDoc of finalMissing) { + if (stored.has(hashDoc.hash)) continue; if (currentHeight - hashDoc.retryFromHeight > HASH_EXPIRY_BLOCKS) { // eslint-disable-next-line no-await-in-loop await messageVerifier.appHashHasMessageNotFound(hashDoc.hash); @@ -817,8 +849,8 @@ async function triggerAppHashesCheckAPI(req, res) { res.json(errMessage); return; } - syncMissingHashes({ force: true }).catch((err) => log.error(`syncMissingHashes API trigger: ${err.message}`)); - const resultsResponse = messageHelper.createSuccessMessage('Running sync on missing application messages'); + appSyncEvents.emit(EVENTS.HASH_CHECK_REQUESTED); + const resultsResponse = messageHelper.createSuccessMessage('Sync on missing application messages queued'); res.json(resultsResponse); } catch (error) { log.error(error); diff --git a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js index c3f29f08da..7a57bc1ab5 100644 --- a/ZelBack/src/services/appMessaging/appSyncOrchestrator.js +++ b/ZelBack/src/services/appMessaging/appSyncOrchestrator.js @@ -115,8 +115,15 @@ class AppSyncOrchestrator { #ephemeralSyncHandler = null; #ephemeralRefusedHandler = null; #ephemeralUnverifiedHandler = null; + + #ephemeralWriteFailedHandler = null; #ephemeralProgressHandler = null; #hashUnresolvedHandler = null; + + #hashCheckRequestedHandler = null; + + // An operator's check of every missing app message, waiting for the node. + #hashCheckRequested = false; #broadcastStarted = null; #started = false; #syncInProgress = false; @@ -164,6 +171,10 @@ class AppSyncOrchestrator { #nextHashRetryHeight = 0; #lastBlockHeight = 0; #fluxVersion = null; + + // Whether this version's once-per-upgrade hash retry reset has run, or was + // already recorded as run. The version marker is written only once it has. + #upgradeResetDone = false; #heartbeatInterval = null; #bootContext = null; #canSendMessages = false; @@ -251,15 +262,24 @@ class AppSyncOrchestrator { this.#ephemeralRefusedHandler = (syncType, peerKey) => this.#onEphemeralSyncRefused(syncType, peerKey); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_REFUSED, this.#ephemeralRefusedHandler); - this.#ephemeralUnverifiedHandler = (peerKey) => this.#onEphemeralSyncUnverified(peerKey); + this.#ephemeralUnverifiedHandler = (peerKey) => this.#endFailedRequest(peerKey, 'unverified', 'sent a response this node could not verify', 'ephemeralSync:peerUnverified'); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_UNVERIFIED, this.#ephemeralUnverifiedHandler); + this.#ephemeralWriteFailedHandler = (peerKey) => this.#endFailedRequest(peerKey, 'writeFailed', 'sent a response this node could not record', 'ephemeralSync:peerWriteFailed'); + appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, this.#ephemeralWriteFailedHandler); + this.#ephemeralProgressHandler = (peerKey) => this.#onEphemeralSyncProgress(peerKey); appSyncEvents.on(EVENTS.EPHEMERAL_SYNC_PROGRESS, this.#ephemeralProgressHandler); this.#hashUnresolvedHandler = () => this.#onHashUnresolved(); appSyncEvents.on(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); + this.#hashCheckRequestedHandler = () => { + this.#hashCheckRequested = true; + this.#runRequestedHashCheck(); + }; + appSyncEvents.on(EVENTS.HASH_CHECK_REQUESTED, this.#hashCheckRequestedHandler); + this.#blockReceivedHandler = (blockHeight) => { this.#onBlocksProcessed(blockHeight); }; @@ -306,6 +326,12 @@ class AppSyncOrchestrator { log.error(`AppSyncOrchestrator - ${syncType} sync complete with no peer, not counted`); return; } + // A message admitted while the request was open can finish after it ended: + // timed out, or ended for an answer that could not be verified or recorded. + if (!this.#requests.isOpen(peerKey)) { + log.info(`AppSyncOrchestrator - ${syncType} sync complete from ${peerKey} after its request ended, not counted`); + return; + } answered.add(peerKey); // Its answer is in, so it is no longer something being waited on. The // record stands as answered rather than being dropped: a peer that has @@ -555,25 +581,26 @@ class AppSyncOrchestrator { } /** - * A peer sent something this node cannot attribute to it. + * End a peer's request whose answer cannot be taken, and ask another peer. * - * Its request ends here rather than on a deadline, because the answer is - * already known: a stream with a hole in it is not a survey, and waiting the - * peer out would spend one of very few slots on an answer that cannot be - * counted whatever else arrives. The record stands, so the peer is not asked - * again on this connection. - * @param {string} peerKey ip:port of the peer whose response failed. - * @returns {void} + * Ended here rather than on a deadline: an answer with a hole in it cannot be + * counted whatever else arrives, and waiting the peer out would spend one of very + * few slots on it. The record stands, so the peer is not asked again on this + * connection. + * @param {string} peerKey + * @param {string} outcome What the request is settled as. + * @param {string} why For the log. + * @param {string} busEvent Published when this ended the request. */ - #onEphemeralSyncUnverified(peerKey) { + #endFailedRequest(peerKey, outcome, why, busEvent) { if (this.#stateSyncComplete) return; if (!peerKey) { - log.error('AppSyncOrchestrator - An unverifiable sync response named no peer, cannot replace it'); + log.error(`AppSyncOrchestrator - A sync response named no peer (${outcome}), cannot replace it`); return; } - if (this.#closeRequest(peerKey, 'unverified')) { - log.warn(`AppSyncOrchestrator - ${peerKey} sent a response this node could not verify, asking another peer`); - fluxEventBus.publish('ephemeralSync:peerUnverified', { peer: peerKey }); + if (this.#closeRequest(peerKey, outcome)) { + log.warn(`AppSyncOrchestrator - ${peerKey} ${why}, asking another peer`); + fluxEventBus.publish(busEvent, { peer: peerKey }); } // Unconditional for the same reason a refusal is: the deficit decides, and // a pool that is already whole asks nobody. @@ -871,6 +898,25 @@ class AppSyncOrchestrator { this.#checkReadiness(); this.#checkHashRetry(blockHeight); } + this.#runRequestedHashCheck(); + } + + /** + * Runs the check an operator asked for, once no hash sync or registry build holds + * the node; a request made while one runs waits for a later block. + * @returns {Promise} + */ + async #runRequestedHashCheck() { + if (!this.#hashCheckRequested || this.#syncInProgress) return; + this.#hashCheckRequested = false; + this.#syncInProgress = true; + try { + await appHashSyncService.syncMissingHashes({ force: true, currentHeight: this.#lastBlockHeight }); + } catch (error) { + log.error(`AppSyncOrchestrator - Requested hash check failed: ${error.message}`); + } finally { + this.#syncInProgress = false; + } } #onHashUnresolved() { @@ -887,15 +933,12 @@ class AppSyncOrchestrator { this.#syncInProgress = true; try { + if (!this.#upgradeResetDone) { + await this.#checkVersionUpgrade(); + await this.#writeVersionMarker(); + } const result = await appHashSyncService.syncMissingHashes({ currentHeight: this.#lastBlockHeight }); this.#nextHashRetryHeight = result.nextRetryHeight ?? (this.#lastBlockHeight + FALLBACK_RECHECK_BLOCKS); - // A retry stores the messages it resolves without promoting them, so nothing else applies - // them: a renewal fetched here would never reach globalAppsInformation, and the app would - // expire on this node alone. Rebuild from the log, as the initial sync does. - if (result.resolved > 0) { - await registryManager.reindexGlobalAppsInformation() - .catch((error) => log.error(`AppSyncOrchestrator - Rebuild after hash retry failed: ${error.message}`)); - } if (result.missing > 0) { log.info(`AppSyncOrchestrator - Hash retry: ${result.resolved} resolved, ${result.missing} remaining, next check at block ${this.#nextHashRetryHeight}`); } @@ -914,7 +957,6 @@ class AppSyncOrchestrator { return; } log.info('AppSyncOrchestrator - Sync started'); - await this.#checkVersionUpgrade(); log.info('AppSyncOrchestrator - Starting initial hash sync'); await this.#runHashSync(); this.#checkReadiness(); @@ -930,13 +972,14 @@ class AppSyncOrchestrator { const resetCount = await appHashSyncService.resetHashSyncForUpgrade(this.#lastBlockHeight); log.info(`AppSyncOrchestrator - Version upgrade to ${this.#fluxVersion}, reset ${resetCount} hash sync entries`); } + this.#upgradeResetDone = true; } catch (error) { log.error(`AppSyncOrchestrator - Version upgrade check failed: ${error.message}`); } } async #writeVersionMarker() { - if (!this.#fluxVersion) return; + if (!this.#fluxVersion || !this.#upgradeResetDone) return; try { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); @@ -956,6 +999,7 @@ class AppSyncOrchestrator { this.#syncInProgress = true; try { this.#hashSyncAttempts += 1; + if (!this.#upgradeResetDone) await this.#checkVersionUpgrade(); const result = await appHashSyncService.syncMissingHashes({ currentHeight: this.#lastBlockHeight }); if (result.missing > 0) { log.warn(`AppSyncOrchestrator - Hash sync has ${result.missing} unresolvable hashes, proceeding`); @@ -1047,7 +1091,16 @@ class AppSyncOrchestrator { // Block timer fired but hash sync / DB rebuild never completed — rebuild from whatever data we have if (blockTimerExpired && !this.#dbRebuilt) { - await this.#rebuildDb(); + // Never beside a hash sync: one storing messages while the registry is unbuilt + // leaves their rows to this build, which would miss any stored after it read + // the log. A build that has to wait is asked again on the next block. + if (this.#syncInProgress) return; + this.#syncInProgress = true; + try { + await this.#rebuildDb(); + } finally { + this.#syncInProgress = false; + } if (!this.#dbRebuilt) return; } @@ -1124,22 +1177,19 @@ class AppSyncOrchestrator { log.info(`Boot context: machineRebooted=${machineRebooted} downtime=${Math.round(downtimeMs / 1000)}s cleanShutdown=${cleanShutdown} firstBoot=${!heartbeat}`); return ctx; } catch (error) { + // Unread is neither a first boot nor a long absence: how long the node was down is + // unknown, so whether its locations expired is decided app by app. log.error(`Failed to read boot context: ${error.message}`); - return { machineRebooted: true, downtimeMs: Infinity, cleanShutdown: false, currentBootId: null, firstBoot: true }; - } - } - - async #clearShutdownReason() { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.local.database); - await dbHelper.findOneAndUpdateInDatabase(database, startupCollection, { _id: 'heartbeat' }, { $unset: { shutdownReason: '' } }); - } catch (error) { - log.error(`Failed to clear shutdown reason: ${error.message}`); + return { + unreadable: true, machineRebooted: true, downtimeMs: null, cleanShutdown: false, currentBootId: null, firstBoot: false, + }; } } #startHeartbeat() { + // The reason the previous process stopped is its own: cleared by the first heartbeat + // that lands, so a crash of this one never reads as that stop. + let shutdownReasonCleared = false; const writeHeartbeat = async () => { try { const db = dbHelper.databaseConnection(); @@ -1148,12 +1198,13 @@ class AppSyncOrchestrator { if (this.#bootContext?.currentBootId) { update.$set.machineBootId = this.#bootContext.currentBootId; } + if (!shutdownReasonCleared) update.$unset = { shutdownReason: '' }; await dbHelper.findOneAndUpdateInDatabase(database, startupCollection, { _id: 'heartbeat' }, update, { upsert: true }); + shutdownReasonCleared = true; } catch (error) { log.error(`Heartbeat write failed: ${error.message}`); } }; - this.#clearShutdownReason(); writeHeartbeat(); this.#heartbeatInterval = setInterval(writeHeartbeat, config.system.heartbeatIntervalMs ?? 30000); } @@ -1204,10 +1255,14 @@ class AppSyncOrchestrator { } if (this.#ephemeralUnverifiedHandler) { appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_UNVERIFIED, this.#ephemeralUnverifiedHandler); + appSyncEvents.removeListener(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, this.#ephemeralWriteFailedHandler); } if (this.#hashUnresolvedHandler) { appSyncEvents.removeListener(EVENTS.HASH_UNRESOLVED, this.#hashUnresolvedHandler); } + if (this.#hashCheckRequestedHandler) { + appSyncEvents.removeListener(EVENTS.HASH_CHECK_REQUESTED, this.#hashCheckRequestedHandler); + } if (this.#blockReceivedHandler) { this.#blockEmitter.removeListener('blocksProcessed', this.#blockReceivedHandler); } diff --git a/ZelBack/src/services/appMessaging/cryptographicKeys.js b/ZelBack/src/services/appMessaging/cryptographicKeys.js index a35775536a..9e9fc5ad09 100644 --- a/ZelBack/src/services/appMessaging/cryptographicKeys.js +++ b/ZelBack/src/services/appMessaging/cryptographicKeys.js @@ -6,6 +6,7 @@ const benchmarkService = require('../benchmarkService'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const log = require('../../lib/log'); const { Privilege, authOf } = require('../utils/privileges'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); // Check if running on Arcane OS const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -67,6 +68,7 @@ async function getPublicKey(req, res) { if (!appSpecification.owner || !appSpecification.name) { throw new Error('Input parameters missing.'); } + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); if (!syncStatus.data.synced) { throw new Error('Daemon not yet synced.'); diff --git a/ZelBack/src/services/appMessaging/messageStore.js b/ZelBack/src/services/appMessaging/messageStore.js index fb59298604..fa09fc8217 100644 --- a/ZelBack/src/services/appMessaging/messageStore.js +++ b/ZelBack/src/services/appMessaging/messageStore.js @@ -9,7 +9,10 @@ const registryManager = require('../appDatabase/registryManager'); const { socketAddressesMatch } = require('../utils/socketAddressUtils'); // const advancedWorkflows = require('../appLifecycle/advancedWorkflows'); // Moved to dynamic require to avoid circular dependency const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const globalState = require('../utils/globalState'); +const { appRunningEntries } = require('../utils/appRunningEntries'); +const fluxEventBus = require('../utils/fluxEventBus'); const { globalAppsMessages, globalAppsTempMessages, @@ -166,6 +169,7 @@ async function storeAppTemporaryMessage(message, options = {}) { // eslint-disable-next-line global-require const fluxService = require('../fluxService'); if (await fluxService.isSystemSecure()) { + verifyAppNameAndOwner(appSpecFormatted.name, appSpecFormatted.owner); // eslint-disable-next-line no-use-before-define const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecFormatted, @@ -289,9 +293,31 @@ function runningSinceOf(isNewer, runningSince, moved) { } /** - * Store app running message + * Log, in one line, the apprunning entries a store refused. + * @param {string} caller The function that refused them. + * @param {Array<{ip: string, broadcastedAt: number, refused: Array<*>}>} refusals One per message with refused entries. + */ +function logRefusedEntries(caller, refusals) { + if (!refusals.length) return; + const entries = refusals.reduce((total, { refused }) => total + refused.length, 0); + const shown = refusals.slice(0, 3).map(({ ip, broadcastedAt, refused }) => ( + `${ip} at ${broadcastedAt}: ${refused.slice(0, 3).map((app) => JSON.stringify(app)?.slice(0, 200)).join(', ')}` + )).join('; '); + log.warn(`${caller} - ${entries} app entries in ${refusals.length} messages have no string name and hash and are not stored. ${shown}`); +} + +/** + * Store app running message. + * + * A version 2 message is stored entry by entry: the entries with a name and a + * hash are written and the rest are refused, logged, and leave the others alone. + * A message whose every entry is refused stores nothing and is not relayed. It + * is never read as an empty snapshot, which deletes the sender's rows: only a + * message that names no apps at all says that. * @param {object} message - Message to store - * @returns {Promise} Whether message should be rebroadcast or Error if invalid + * @returns {Promise<{stored: boolean, rebroadcast: boolean, written: Array, refused: Array<*>, cleared: boolean}|Error>} + * Whether anything was stored and the message is to be relayed, the entries written, the names + * refused, and whether the sender's rows were cleared; or Error when the message itself is malformed */ async function storeAppRunningMessage(message) { /* message object @@ -307,6 +333,9 @@ async function storeAppRunningMessage(message) { * @param apps array (for version 2) */ const appsMessages = []; + // What this message did here, for the caller to report: the entries written, + // the names refused, and whether the sender's rows were cleared. + let refusedNames = []; if (!message || typeof message !== 'object' || typeof message.type !== 'string' || typeof message.version !== 'number' || typeof message.broadcastedAt !== 'number' || typeof message.ip !== 'string') { return new Error('Invalid Flux App Running message for storing'); @@ -334,18 +363,24 @@ async function storeAppRunningMessage(message) { if (message.apps.length > config.fluxapps.maxAppsPerNode) { return new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode'); } - for (let i = 0; i < message.apps.length; i += 1) { - const app = message.apps[i]; - appsMessages.push(app); - if (typeof app.hash !== 'string' || typeof app.name !== 'string') { - return new Error('Invalid Flux App Running v2 message for storing'); - } + const { stored, refused } = appRunningEntries(message.apps); + refusedNames = refused.map((app) => app?.name ?? null); + if (refused.length) { + logRefusedEntries('storeAppRunningMessage', [{ ip: message.ip, broadcastedAt: message.broadcastedAt, refused }]); } + if (message.apps.length && !stored.length) { + return { + stored: false, rebroadcast: false, written: [], refused: refusedNames, cleared: false, + }; + } + appsMessages.push(...stored); } if (message.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) { log.warn(`Rejecting old/not valid Fluxapprunning message, message:${JSON.stringify(message)}`); - return { stored: false, rebroadcast: false }; + return { + stored: false, rebroadcast: false, written: [], refused: refusedNames, cleared: false, + }; } const db = dbHelper.databaseConnection(); @@ -354,6 +389,7 @@ async function storeAppRunningMessage(message) { const [ip] = await registryManager.currentAddressesOf([{ ip: message.ip, at: message.broadcastedAt }]); let anyStored = false; + const written = []; const incomingDate = new Date(message.broadcastedAt); const isNewer = { $gt: [incomingDate, { $ifNull: ['$broadcastedAt', new Date(0)] }] }; @@ -383,6 +419,7 @@ async function storeAppRunningMessage(message) { ); if (result.modifiedCount > 0 || result.upsertedCount > 0) { anyStored = true; + written.push(app); } } @@ -396,7 +433,9 @@ async function storeAppRunningMessage(message) { await dbHelper.removeDocumentsFromCollection(database, appsInstallingBroadcasts, { 'data.ip': message.ip }); anyStored = true; } else { - return { stored: false, rebroadcast: false }; + return { + stored: false, rebroadcast: false, written: [], refused: refusedNames, cleared: false, + }; } } @@ -408,7 +447,13 @@ async function storeAppRunningMessage(message) { await dbHelper.removeDocumentsFromCollection(database, appsInstallingBroadcasts, { 'data.name': app.name, 'data.ip': message.ip }); } - return { stored: anyStored, rebroadcast: anyStored }; + return { + stored: anyStored, + rebroadcast: anyStored, + written, + refused: refusedNames, + cleared: message.version === 2 && appsMessages.length === 0 && anyStored, + }; } /** @@ -766,7 +811,7 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); - const { stored } = await storeBatchAppRunningEvents(verifiedBroadcasts); + const { stored, writeFailed: eventsFailed } = await storeBatchAppRunningEvents(verifiedBroadcasts); // One operation is built per app per broadcast and each carries a merge // pipeline, so a sync response of a few thousand broadcasts expands into tens @@ -774,7 +819,7 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { // the driver's encoding of it small; the operations are independent of one // another, which is what makes the unordered write safe to split. const locationOps = []; - let writeFailed = false; + let writeFailed = eventsFailed; const flushLocationOps = async () => { if (!locationOps.length) return; const batch = locationOps.splice(0, locationOps.length); @@ -789,13 +834,17 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { verifiedBroadcasts.map(({ data }) => ({ ip: data.ip, at: data.broadcastedAt })), ); + // one log line per call: a sync response is stored a slice at a time, and a + // slice holds thousands of broadcasts + const refusals = []; for (const [index, broadcast] of verifiedBroadcasts.entries()) { const { data } = broadcast; const validTill = data.broadcastedAt + RUNNING_EXPIRY_MS; if (validTill < Date.now()) continue; const ip = addresses[index]; - const apps = data.version === 2 ? (data.apps || []) : [{ name: data.name, hash: data.hash }]; + const { stored: apps, refused } = appRunningEntries(data.version === 2 ? (data.apps || []) : [{ name: data.name, hash: data.hash }]); + if (refused.length) refusals.push({ ip: data.ip, broadcastedAt: data.broadcastedAt, refused }); const incomingDate = new Date(data.broadcastedAt); const incomingExpiry = new Date(validTill); const isNewer = { $gt: [incomingDate, { $ifNull: ['$broadcastedAt', new Date(0)] }] }; @@ -829,6 +878,13 @@ async function storeBatchAppRunningMessages(verifiedBroadcasts) { } await flushLocationOps(); + logRefusedEntries('storeBatchAppRunningMessages', refusals); + if (refusals.length) { + fluxEventBus.publish('network:apprunningRefused', { + path: 'sync', + refusals: refusals.map(({ ip, refused }) => ({ ip, names: refused.map((app) => app?.name ?? null) })), + }); + } // Upserts only. Removing the rows a node no longer reports belongs to // pruneAppRunningLocations, which is driven from the newest broadcast per node @@ -858,8 +914,7 @@ async function pruneAppRunningLocations(newestByIp) { const flush = async () => { if (!ops.length) return; const batch = ops.splice(0, ops.length); - await database.collection(globalAppsLocations).bulkWrite(batch, { ordered: false }) - .catch((err) => log.error(`pruneAppRunningLocations: ${err.message}`)); + await database.collection(globalAppsLocations).bulkWrite(batch, { ordered: false }); }; const newest = [...newestByIp]; @@ -882,77 +937,78 @@ async function pruneAppRunningLocations(newestByIp) { // --- Event Log Functions --- -async function handleAppRunningEvent({ signedBroadcast }) { - try { - const { data } = signedBroadcast; - if (!data || !data.ip || !data.broadcastedAt) return; - if (data.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) return; +/** + * Whether an apprunning message names apps and none of them can be stored. Such a + * message says nothing this node can act on, and is kept out of the event log so + * it never takes the place of the sender's last announcement there. + * @param {object} data The message's data. + * @returns {boolean} + */ +function namesOnlyRefusedApps(data) { + return data.version === 2 && Array.isArray(data.apps) && data.apps.length > 0 + && appRunningEntries(data.apps).stored.length === 0; +} - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); +async function handleAppRunningEvent({ signedBroadcast }) { + const { data } = signedBroadcast; + if (!data || !data.ip || !data.broadcastedAt) return; + if (data.broadcastedAt + GOSSIP_VALIDITY_MS < Date.now()) return; + if (namesOnlyRefusedApps(data)) return; - if (data.version === 2 && (!data.apps || data.apps.length === 0)) { - const existing = await database.collection(globalAppStateEvents).findOne({ ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING }); - if (!existing) return; - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); - const dedupKey = data.apps ? 'v2' : `v1:${data.name}`; - const envelope = { version: signedBroadcast.version, timestamp: signedBroadcast.timestamp, pubKey: signedBroadcast.pubKey, signature: signedBroadcast.signature }; - - await database.collection(globalAppStateEvents).updateOne( - { ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey }, - buildConditionalUpsert(data.broadcastedAt, { - ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey, - broadcastedAt: new Date(data.broadcastedAt), - expireAt: new Date(data.broadcastedAt + RUNNING_EXPIRY_MS), - data, envelope, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(apprunning): ${err.message}`); + if (data.version === 2 && (!data.apps || data.apps.length === 0)) { + const existing = await database.collection(globalAppStateEvents).findOne({ ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING }); + if (!existing) return; } + + const dedupKey = data.apps ? 'v2' : `v1:${data.name}`; + const envelope = { version: signedBroadcast.version, timestamp: signedBroadcast.timestamp, pubKey: signedBroadcast.pubKey, signature: signedBroadcast.signature }; + + await database.collection(globalAppStateEvents).updateOne( + { ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey }, + buildConditionalUpsert(data.broadcastedAt, { + ip: data.ip, type: APP_STATE_EVENT_TYPES.APPRUNNING, dedupKey, + broadcastedAt: new Date(data.broadcastedAt), + expireAt: new Date(data.broadcastedAt + RUNNING_EXPIRY_MS), + data, envelope, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } async function handleSigtermEvent({ message, envelope }) { if (!message || !message.ip || !message.broadcastedAt) return; - try { - const { ip } = message; - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm' }, - buildConditionalUpsert(message.broadcastedAt, { - ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm', - broadcastedAt: new Date(message.broadcastedAt), - expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), - envelope, data: message, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(sigterm): ${err.message}`); - } + const { ip } = message; + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm' }, + buildConditionalUpsert(message.broadcastedAt, { + ip, type: APP_STATE_EVENT_TYPES.SIGTERM, dedupKey: 'sigterm', + broadcastedAt: new Date(message.broadcastedAt), + expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), + envelope, data: message, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } async function handleAppRemovedStateEvent({ message, envelope }) { if (!message || !message.ip || !message.appName || !message.broadcastedAt) return; - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}` }, - buildConditionalUpsert(message.broadcastedAt, { - ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}`, - broadcastedAt: new Date(message.broadcastedAt), - expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), - envelope, data: message, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(appremoved): ${err.message}`); - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}` }, + buildConditionalUpsert(message.broadcastedAt, { + ip: message.ip, type: APP_STATE_EVENT_TYPES.APPREMOVED, dedupKey: `appremoved:${message.appName}`, + broadcastedAt: new Date(message.broadcastedAt), + expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), + envelope, data: message, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } /** @@ -984,20 +1040,16 @@ function evictionTime(evictedAt) { async function handleEvictedEvent({ ip, evictedAt }) { if (!ip) return; const at = evictionTime(evictedAt); - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip, type: APP_STATE_EVENT_TYPES.EVICTED, dedupKey: 'evicted' }, - { - $set: { receivedAt: new Date() }, - $max: { createdAt: new Date(at), expireAt: new Date(at + EVICTED_EXPIRY_MS) }, - }, - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(evicted): ${err.message}`); - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip, type: APP_STATE_EVENT_TYPES.EVICTED, dedupKey: 'evicted' }, + { + $set: { receivedAt: new Date() }, + $max: { createdAt: new Date(at), expireAt: new Date(at + EVICTED_EXPIRY_MS) }, + }, + { upsert: true }, + ); } /** @@ -1027,22 +1079,18 @@ async function applyEviction(ip, evictedAt) { async function handleIPChangedEvent({ message, envelope }) { if (!message || !message.oldIP || !message.newIP || !message.broadcastedAt) return; - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - await database.collection(globalAppStateEvents).updateOne( - { ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged' }, - buildConditionalUpsert(message.broadcastedAt, { - ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged', - broadcastedAt: new Date(message.broadcastedAt), - expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), - data: message, envelope: envelope ?? null, - }, { alwaysSetFields: { receivedAt: new Date() } }), - { upsert: true }, - ); - } catch (err) { - log.error(`storeAppStateEvent(ipchanged): ${err.message}`); - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.appsglobal.database); + await database.collection(globalAppStateEvents).updateOne( + { ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged' }, + buildConditionalUpsert(message.broadcastedAt, { + ip: message.oldIP, type: APP_STATE_EVENT_TYPES.IPCHANGED, dedupKey: 'ipchanged', + broadcastedAt: new Date(message.broadcastedAt), + expireAt: new Date(message.broadcastedAt + RUNNING_EXPIRY_MS), + data: message, envelope: envelope ?? null, + }, { alwaysSetFields: { receivedAt: new Date() } }), + { upsert: true }, + ); } function storeAppStateEvent(type, payload) { @@ -1068,6 +1116,7 @@ async function storeBatchAppRunningEvents(verifiedBroadcasts) { if (!data || !data.ip || !data.broadcastedAt) continue; const validTill = data.broadcastedAt + RUNNING_EXPIRY_MS; if (validTill < Date.now()) continue; + if (namesOnlyRefusedApps(data)) continue; const dedupKey = data.apps ? 'v2' : `v1:${data.name}`; const envelope = { version: broadcast.version, timestamp: broadcast.timestamp, pubKey: broadcast.pubKey, signature: broadcast.signature }; @@ -1087,12 +1136,16 @@ async function storeBatchAppRunningEvents(verifiedBroadcasts) { } + let writeFailed = false; if (ops.length > 0) { await database.collection(globalAppStateEvents).bulkWrite(ops, { ordered: false }) - .catch((err) => log.error(`storeBatchAppRunningEvents: ${err.message}`)); + .catch((err) => { + writeFailed = true; + log.error(`storeBatchAppRunningEvents: ${err.message}`); + }); } - return { stored: ops.length }; + return { stored: ops.length, writeFailed }; } function storeSignedAppInstallingBroadcast(signedBroadcast) { @@ -1116,11 +1169,16 @@ function storeSignedAppInstallingBroadcast(signedBroadcast) { { 'data.name': data.name, 'data.ip': data.ip }, { $set: doc }, { upsert: true }, - ).catch((err) => log.error(`storeSignedAppInstallingBroadcast: ${err.message}`)); + ); } async function storeBatchAppInstallingMessages(verifiedBroadcasts) { - if (verifiedBroadcasts.length === 0) return { stored: 0 }; + if (verifiedBroadcasts.length === 0) return { stored: 0, writeFailed: false }; + let writeFailed = false; + const failed = (what, err) => { + writeFailed = true; + log.error(`storeBatchAppInstallingMessages ${what}: ${err.message}`); + }; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -1189,17 +1247,17 @@ async function storeBatchAppInstallingMessages(verifiedBroadcasts) { if (signedOps.length > 0) { await database.collection(appsInstallingBroadcasts).bulkWrite(signedOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingMessages signed: ${err.message}`)); + .catch((err) => failed('signed', err)); } if (locationOps.length > 0) { await database.collection(globalAppsInstallingLocations).bulkWrite(locationOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingMessages locations: ${err.message}`)); + .catch((err) => failed('locations', err)); } // after the claims, so a claim and its withdrawal arriving in one batch settle // in the order they were sent rather than the order they were read if (withdrawalOps.length > 0) { await database.collection(globalAppsInstallingLocations).bulkWrite(withdrawalOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingMessages withdrawals: ${err.message}`)); + .catch((err) => failed('withdrawals', err)); await database.collection(appsInstallingBroadcasts).bulkWrite( // The location's own guard, carried over rather than restated: the // broadcast is what proves the location, so deleting one without the @@ -1215,9 +1273,9 @@ async function storeBatchAppInstallingMessages(verifiedBroadcasts) { }, })), { ordered: false }, - ).catch((err) => log.error(`storeBatchAppInstallingMessages withdrawal broadcasts: ${err.message}`)); + ).catch((err) => failed('withdrawal broadcasts', err)); } - return { stored: signedOps.length + withdrawalOps.length }; + return { stored: signedOps.length + withdrawalOps.length, writeFailed }; } function storeSignedAppInstallingErrorBroadcast(signedBroadcast) { @@ -1241,11 +1299,16 @@ function storeSignedAppInstallingErrorBroadcast(signedBroadcast) { { 'data.name': data.name, 'data.hash': data.hash, 'data.ip': data.ip }, { $set: doc }, { upsert: true }, - ).catch((err) => log.error(`storeSignedAppInstallingErrorBroadcast: ${err.message}`)); + ); } async function storeBatchAppInstallingErrorMessages(verifiedBroadcasts) { - if (verifiedBroadcasts.length === 0) return { stored: 0 }; + if (verifiedBroadcasts.length === 0) return { stored: 0, writeFailed: false }; + let writeFailed = false; + const failed = (what, err) => { + writeFailed = true; + log.error(`storeBatchAppInstallingErrorMessages ${what}: ${err.message}`); + }; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -1298,13 +1361,13 @@ async function storeBatchAppInstallingErrorMessages(verifiedBroadcasts) { if (signedOps.length > 0) { await database.collection(globalAppsInstallingErrorsBroadcasts).bulkWrite(signedOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingErrorMessages signed: ${err.message}`)); + .catch((err) => failed('signed', err)); } if (locationOps.length > 0) { await database.collection(globalAppsInstallingErrorsLocations).bulkWrite(locationOps, { ordered: false }) - .catch((err) => log.error(`storeBatchAppInstallingErrorMessages locations: ${err.message}`)); + .catch((err) => failed('locations', err)); } - return { stored: signedOps.length }; + return { stored: signedOps.length, writeFailed }; } module.exports = { diff --git a/ZelBack/src/services/appMessaging/messageVerifier.js b/ZelBack/src/services/appMessaging/messageVerifier.js index 14825d65c6..c39487e1d9 100644 --- a/ZelBack/src/services/appMessaging/messageVerifier.js +++ b/ZelBack/src/services/appMessaging/messageVerifier.js @@ -574,15 +574,24 @@ async function storeGoverningAppSpecifications(appNames) { } /** - * Moves each stored message to this node's record of its payment (moveStoredMessagesToPayments), - * and writes the registry row of each app with a message that moved - * (storeGoverningAppSpecifications). + * Resolves messages this node already holds: moves each to this node's record of its payment + * (moveStoredMessagesToPayments), and writes the registry row of every app they belong to + * (storeGoverningAppSpecifications). A message whose promotion stopped before its registry row + * was written is completed here, so the row is written before the hash is marked as resolved. * @param {Array<{hash: string, txid: string, height: number, value: number}>} records * @returns {Promise} the names of the apps with a message that moved */ async function alignStoredMessagesWithPayments(records) { const moved = await moveStoredMessagesToPayments(records); - await storeGoverningAppSpecifications(moved); + if (records.length === 0) return moved; + const appsDatabase = dbHelper.databaseConnection().db(config.database.appsglobal.database); + const stored = await dbHelper.findInDatabase( + appsDatabase, + globalAppsMessages, + { hash: { $in: records.map((record) => record.hash) } }, + { projection: { _id: 0, 'appSpecifications.name': 1 } }, + ); + await storeGoverningAppSpecifications([...new Set(stored.map((message) => message.appSpecifications.name))]); return moved; } @@ -753,10 +762,17 @@ async function paymentRecordOf(hash) { * @param {number} height * @param {number} valueSat * @param {number} [i] attempt - * @returns {Promise} whether the message is stored + * @returns {Promise} whether the message is stored. False when the payment record + * could not be read: the hash stays unresolved and is asked about again. */ async function checkAndRequestApp(hash, txid, height, valueSat, i = 0) { - const record = await paymentRecordOf(hash); + let record; + try { + record = await paymentRecordOf(hash); + } catch (error) { + log.error(`checkAndRequestApp - payment record for ${hash} could not be read, left unresolved: ${error.message}`); + return false; + } if (!record) { log.warn(`checkAndRequestApp - no payment recorded for ${hash} (requested from txid ${txid}), not stored`); return false; @@ -851,8 +867,10 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { // eslint-disable-next-line global-require const messageStore = require('./messageStore'); await messageStore.storeAppPermanentMessage(permanentAppMessage); - // await update zelapphashes that we already have it stored - await appHashHasMessage(hash); + // The hash is marked as holding its message only once the message's effect on + // the registry is written, at each way out below: a failure before that leaves + // the hash unresolved, and resolving it again writes the registry + // (alignStoredMessagesWithPayments). const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); const daemonHeight = syncStatus.data.height; @@ -897,12 +915,8 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { updateForSpecifications.hash = permanentAppMessage.hash; updateForSpecifications.height = permanentAppMessage.height; // object of appSpecifications extended for hash and height - const inserted = await storeAppSpecificationInForce(updateForSpecifications); + await storeAppSpecificationInForce(updateForSpecifications); const appName = specifications.name; - if (!inserted) { - globalState.clearPendingUpdates(appName); - return true; - } const pendingUpdates = globalState.getPendingUpdates(appName); if (pendingUpdates.length > 0) { log.info(`Processing ${pendingUpdates.length} pending updates for ${appName}`); @@ -939,6 +953,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { ); if (!messageInfo) { log.error(`Last permanent message for ${specifications.name} not found`); + await appHashHasMessage(hash); return true; } const previousSpecs = messageInfo.appSpecifications; @@ -984,6 +999,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { log.warn(`Apps message ${permanentAppMessage.hash} is underpaid ${valueSat} < ${appPrice * 1e8}`); } } + await appHashHasMessage(hash); return true; } else { // App has expired (actualExpirationHeight <= daemonHeight) @@ -995,6 +1011,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { // ending the app here would drop a paid app from this node and uninstall it. if (!await isGoverningAppMessage(specifications.name, permanentAppMessage.hash)) { log.info(`App message ${permanentAppMessage.hash} for ${specifications.name} has expired, but does not govern the app. Nothing to clean up.`); + await appHashHasMessage(hash); return true; } log.warn(`App ${specifications.name} has expired (expiration height ${actualExpirationHeight} <= daemon height ${daemonHeight}). Cleaning up stale data.`); @@ -1025,6 +1042,7 @@ async function checkAndRequestRecordedApp(hash, txid, height, valueSat, i = 0) { await appUninstaller.removeAppLocally(specifications.name, null, true, false, true); } + await appHashHasMessage(hash); return true; } } diff --git a/ZelBack/src/services/appMessaging/peerNotification.js b/ZelBack/src/services/appMessaging/peerNotification.js index bd988ef6fc..7737b4b44b 100644 --- a/ZelBack/src/services/appMessaging/peerNotification.js +++ b/ZelBack/src/services/appMessaging/peerNotification.js @@ -13,6 +13,8 @@ const appReconciler = require('../appMonitoring/appReconciler'); const fluxEventBus = require('../utils/fluxEventBus'); const { nodeSigner } = require('../utils/nodeSigner'); +const { appRunningEntries } = require('../utils/appRunningEntries'); +const appPlacements = require('../appManagement/appPlacements'); const { ANNOUNCE_INTERVAL_MS } = require('../utils/appConstants'); const globalAppsLocations = config.database.appsglobal.collections.appsLocations; @@ -33,11 +35,14 @@ let overrunning = false; // loop survives its own abort. Nothing resets this. let broadcasting = false; -// Held for the whole of a cycle, so a stop can wait for the cycle in flight -// rather than returning while it is still running. A teardown that returns -// before the thing is torn down is the same lie as a stop that does not stop. -// A removal waits on it too, which is why it lives in globalState. +// Held from a cycle's snapshot to its broadcast. A removal waits on it, so its +// message goes out after an announcement that took its list before the removal, +// which is why it lives in globalState. const cycleLock = globalState.announceCycle; +// Settles when the cycle in flight, if any, has finished, so a stop can wait for +// it rather than returning while it is still running. A teardown that returns +// before the thing is torn down is the same lie as a stop that does not stop. +let cycleFinished = Promise.resolve(); /** * Schedule the next announcement so that the PERIOD is fixed, rather than the @@ -113,7 +118,7 @@ async function stopBroadcasting() { // The queued repeat goes with the stop. Left set, it is a cycle the next // start would run before the one it schedules for itself. rebroadcastNeeded = false; - await cycleLock.waitReady(); + await cycleFinished; } function initialize() { @@ -132,13 +137,15 @@ async function checkAndNotifyPeersOfRunningApps() { return; } broadcastInProgress = true; + let finishCycle; + cycleFinished = new Promise((resolve) => { finishCycle = resolve; }); // Taken before any work, so the schedule below subtracts the WHOLE cycle - // including the paths that give up early, which cost time too. const startedAt = process.hrtime.bigint(); - await cycleLock.enable(); + let locked = false; try { - // Observation only. The lock above is what a caller waiting on a cycle in - // flight actually waits for. + // Observation only. The lock taken below is what a caller waiting on a cycle + // in flight actually waits for. fluxEventBus.publish('app:announcing', {}); if (!nodeConfirmationService.canSendMessages()) { log.info('checkAndNotifyPeersOfRunningApps - Node cannot send messages, skipping broadcast'); @@ -151,11 +158,16 @@ async function checkAndNotifyPeersOfRunningApps() { // best-effort. Resolves immediately in steady state; capped reconciler-side, // so a wedged reconcile cannot block the node's network presence. await appReconciler.waitForBootDrainSettled(); + // Every app installed at start has a record before the first snapshot reads them. + await appPlacements.waitForBackfill(); - const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); - if (!localSocketAddr) { - throw new Error('Unable to detect Flux IP address'); - } + const localSocketAddr = await fluxNetworkHelper.waitForLocalSocketAddress(); + + // A cycle is in flight from the snapshot to the broadcast, and a removal waits + // for it so the two go out in the order they happened. The waits above are + // outside it, so a node waiting for its address holds up no removal. + await cycleLock.enable(); + locked = true; // Stamped where the snapshot is taken, not where the message is built: every // consumer reads broadcastedAt as the time these apps were installed here, and @@ -188,31 +200,51 @@ async function checkAndNotifyPeersOfRunningApps() { // installed until the removal finishes, and naming it here would re-create the // location row the removal just cleared. An app this node is only testing is // excluded too: its row is thrown away at the end of the test, and the teardown - // that throws it away tells the network nothing. + // that throws it away tells the network nothing. So is an app boot has not yet + // confirmed this node still holds: naming it would renew a location record that + // may have expired, and the boot check would then read it as valid. const applicationsToBroadcast = appsInstalled.filter( (application) => !globalState.departingApps.has(application.name) - && !globalState.testInstallingApps.has(application.name), + && !globalState.testInstallingApps.has(application.name) + && !globalState.unconfirmedApps.has(application.name), ); + // An installed app without a string name and hash is left out and logged: no + // node stores such an entry, and a node that has not upgraded refuses the + // whole message for it, which would cost this node every other app it holds. + const { stored: announceable, refused } = appRunningEntries(applicationsToBroadcast); + if (refused.length) { + log.warn(`checkAndNotifyPeersOfRunningApps - installed apps without a name and hash are not announced: ${refused.map((application) => application?.name).join(', ')}`); + } const apps = []; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); try { + // When this node took each app comes from its own record of that, never from + // the network's copy of its location rows: those lapse in normal running + // while the app goes on running here. An app with no record is announced + // from its row, and from now without one. + const placements = await appPlacements.placementsOf(announceable.map((application) => application.name)); + const unrecorded = []; // eslint-disable-next-line no-restricted-syntax - for (const application of applicationsToBroadcast) { - const queryFind = { name: application.name, ip: localSocketAddr }; - const projection = { _id: 0, runningSince: 1 }; - // eslint-disable-next-line no-await-in-loop - const result = await dbHelper.findOneInDatabase(database, globalAppsLocations, queryFind, projection); - let runningOnMyNodeSince = new Date().toISOString(); - if (result && result.runningSince) { - runningOnMyNodeSince = result.runningSince; + for (const application of announceable) { + let runningOnMyNodeSince = placements.get(application.name); + if (!runningOnMyNodeSince) { + unrecorded.push(application.name); + const queryFind = { name: application.name, ip: localSocketAddr }; + const projection = { _id: 0, runningSince: 1 }; + // eslint-disable-next-line no-await-in-loop + const result = await dbHelper.findOneInDatabase(database, globalAppsLocations, queryFind, projection); + runningOnMyNodeSince = result?.runningSince ?? new Date(); } apps.push({ name: application.name, hash: application.hash, - runningSince: runningOnMyNodeSince, + runningSince: new Date(runningOnMyNodeSince).toISOString(), }); } + if (unrecorded.length) { + log.warn(`checkAndNotifyPeersOfRunningApps - no placement record for ${unrecorded.join(', ')}; announced from the location table`); + } // An empty snapshot is NEVER broadcast: the receive side treats an empty // v2 message as "delete every appsLocations row for this IP" - and we // store our own message first, so it would erase our own presence. Every @@ -240,10 +272,17 @@ async function checkAndNotifyPeersOfRunningApps() { log.warn('checkAndNotifyPeersOfRunningApps - this node cannot sign as itself; its running apps are not announced'); return; } - await messageStore.storeAppRunningMessage(appRunningMessage); + // Every peer applies the same rule to the message as this store does, so a + // message this node will not store is one no peer stores either. + const ownStore = await messageStore.storeAppRunningMessage(appRunningMessage); + if (ownStore instanceof Error) { + log.error(`checkAndNotifyPeersOfRunningApps - this node refuses its own announcement, not broadcast: ${ownStore.message}`); + fluxEventBus.publish('app:announcementRefused', { reason: ownStore.message }); + return; + } const signed = await fluxCommunicationMessagesSender.broadcastMessageToAll(appRunningMessage); await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed }); - fluxEventBus.publish('app:running', { apps, ip: appRunningMessage.ip }); + fluxEventBus.publish('app:running', { apps, ip: appRunningMessage.ip, unannounced: refused.map((application) => application?.name ?? null) }); log.info(`App Running Message broadcasted: ${apps.length} apps`); } catch (err) { log.error(err); @@ -259,7 +298,8 @@ async function checkAndNotifyPeersOfRunningApps() { log.error(error); } finally { broadcastInProgress = false; - cycleLock.disable(); + if (locked) cycleLock.disable(); + finishCycle(); if (rebroadcastNeeded) { rebroadcastNeeded = false; setImmediate(() => checkAndNotifyPeersOfRunningApps()); diff --git a/ZelBack/src/services/appMonitoring/appReconciler.js b/ZelBack/src/services/appMonitoring/appReconciler.js index b97f2249b8..40728bd72d 100644 --- a/ZelBack/src/services/appMonitoring/appReconciler.js +++ b/ZelBack/src/services/appMonitoring/appReconciler.js @@ -375,6 +375,10 @@ function isManagedElsewhere(identifier) { const restore = globalState.restoreInProgress || []; if (backup.includes(mainAppName) || restore.includes(mainAppName)) return true; if (globalState.stoppingContainers.has(dockerService.getAppIdentifier(identifier))) return true; + // an app being removed, including one whose removal is still cleaning up + if (globalState.departingApps.has(mainAppName)) return true; + // an app boot has not yet confirmed this node still holds + if (globalState.unconfirmedApps.has(mainAppName)) return true; return false; } @@ -382,11 +386,8 @@ function isManagedElsewhere(identifier) { * What this component should be doing, why, and - when the answer is an operator * stop - whether that operator asked for a hard kill. * - * The lock and the force flag are fields of one document, so they come from one - * read. The stop branch used to re-read the same document for the flag alone, - * and getState returns null for a read failure exactly as it does for "no - * record" - so a second read that failed turned "kill now" into a drain. One - * read is a window that cannot open, and one round-trip fewer on every stop. + * The lock and the force flag come from one read. A read that fails rejects, + * and the pass is retried without acting: an unread lock is not a lifted one. * * @returns {Promise<{desired: boolean|null, reason: string, force: boolean}>} */ @@ -485,6 +486,13 @@ async function recreateMissing(identifier) { scheduleRetry(identifier, MANAGED_RETRY_MS); return; } + // A database that did not answer says nothing about the container: the + // recreate never got as far as building it. + if (dbHelper.isDatabaseError(err)) { + log.warn(`appReconciler - recreate of ${identifier} could not read what it needs (${err.message}); retrying, not removing`); + scheduleRetry(identifier, MANAGED_RETRY_MS); + return; + } log.error(`appReconciler - failed to recreate ${identifier}: ${err.message}`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'recreateFailed', reason: err.message }); await appTamperingDetectionService.recordEvent(mainAppName, 'recreation_failed', `Container recreation failure: ${err.message}`); @@ -739,6 +747,7 @@ async function healDetachedNetwork(identifier, mainAppName, spec) { async function reconcile(rawIdentifier) { const identifier = canonical(rawIdentifier); if (isManagedElsewhere(identifier)) { + fluxEventBus.count('reconciler:deferred', identifier, 'managedElsewhere'); scheduleRetry(identifier, MANAGED_RETRY_MS); return; } @@ -1106,8 +1115,9 @@ async function reconcile(rawIdentifier) { // without it - which is how far a component has escalated, and whether it // ever went backwards. Read on the backoff path only, which is a paced // restart and therefore cold. - const backoffState = await appsRuntimeState.getState(identifier); - const rung = ((backoffState && backoffState.restartHistory) || []).length; + // For the report only: a failed read leaves the rung unknown, never the wait. + const backoffState = await appsRuntimeState.getState(identifier).catch(() => undefined); + const rung = backoffState === undefined ? null : ((backoffState && backoffState.restartHistory) || []).length; log.warn(`appReconciler - ${identifier} stopped, ${cause}; backing off ${Math.round(wait / 1000)}s before restart (rung ${rung})`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'backoff', waitMs: wait, rung, crashed, @@ -1122,6 +1132,13 @@ async function reconcile(rawIdentifier) { const isComponent = spec.appSpec.version >= 4 && Array.isArray(spec.appSpec.compose); await volumeService.ensureMountPathsExist(spec.comp, mainAppName, isComponent, isComponent ? spec.appSpec : null); + // The restart this start satisfies is the one requested before it: read now, so + // a failed read defers before anything is started, and a restart requested + // while the start runs is still owed afterwards. + const startingState = await appsRuntimeState.getState(identifier); + const pendingGeneration = (startingState && startingState.restartGeneration) || 0; + const satisfiesRestart = pendingGeneration > ((startingState && startingState.actuatedRestartGeneration) || 0); + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.RECONCILER_BEFORE_START, identifier); // A decider writes its opinion outside this single-flight, so one may have been // written during the awaits above; starting on what this pass read would undo it @@ -1159,9 +1176,6 @@ async function reconcile(rawIdentifier) { // A restart of a container that was already stopped IS this start, so the // request is satisfied here. Left pending, the pass that next finds it running // would bounce a container the operator has just watched come up. - const startedState = await appsRuntimeState.getState(identifier); - const pendingGeneration = (startedState && startedState.restartGeneration) || 0; - const satisfiesRestart = pendingGeneration > ((startedState && startedState.actuatedRestartGeneration) || 0); log.info(`appReconciler - ${identifier} restarted`); fluxEventBus.publish('reconciler:actuated', { identifier, action: 'started', exitCode: actual.exitCode }); // A start is exactly when a container can come up attached to no network (a stale diff --git a/ZelBack/src/services/appMonitoring/availabilityChecker.js b/ZelBack/src/services/appMonitoring/availabilityChecker.js index ae688e351a..6960f118a1 100644 --- a/ZelBack/src/services/appMonitoring/availabilityChecker.js +++ b/ZelBack/src/services/appMonitoring/availabilityChecker.js @@ -1,4 +1,5 @@ // Availability Checker - Checks apps availability by testing ports from external nodes +const crypto = require('node:crypto'); const axios = require('axios'); const config = require('config'); const serviceHelper = require('../serviceHelper'); @@ -8,6 +9,11 @@ const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const upnpService = require('../upnpService'); const networkStateService = require('../networkStateService'); const fluxHttpTestServer = require('../utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../utils/fluxUdpTestServer'); +const portManager = require('../appNetwork/portManager'); +const registryManager = require('../appDatabase/registryManager'); +const policyStore = require('../policyStore'); +const fluxEventBus = require('../utils/fluxEventBus'); const { decryptEnterpriseApps } = require('../appQuery/appQueryService'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); @@ -18,6 +24,9 @@ async function handleTestShutdown(testingPort, testHttpServer, isArcane, options const skipFirewall = options.skipFirewall || false; const skipUpnp = options.skipUpnp || false; const skipHttpServer = options.skipHttpServer || false; + const protocols = options.protocols || ['tcp', 'udp']; + + if (options.udpTestServer) options.udpTestServer.close(); const updateFirewall = skipFirewall ? false @@ -25,9 +34,13 @@ async function handleTestShutdown(testingPort, testHttpServer, isArcane, options || await fluxNetworkHelper.isFirewallActive().catch(() => true); if (updateFirewall) { - await fluxNetworkHelper - .deleteTestPortRule(testingPort) - .catch((e) => log.error(e)); + // eslint-disable-next-line no-restricted-syntax + for (const protocol of protocols) { + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper + .deleteTestPortRule(testingPort, protocol) + .catch((e) => log.error(e)); + } } if (!skipUpnp) { @@ -97,6 +110,113 @@ function releaseOwnDos(dosState) { dosState.dosStateValue = 0; } +/** + * The signed policy document that lets a port found closed over UDP count + * toward this check's DOS. Absent or anything but `true`, a confirmed UDP + * failure is reported in the node's health and counts nothing. + */ +const UDP_DOS_DOCUMENT = 'udpportdos'; + +/** + * Whether a port found closed over UDP counts toward this check's DOS. + * @returns {boolean} + */ +function udpDosEnabled() { + return policyStore.getDocument(UDP_DOS_DOCUMENT) === true; +} + +/** + * How many passes a port is tested again over UDP, looking for a peer that + * probes UDP - to confirm one peer's silence, or on a UDP pass that learned + * nothing - before the pass is left unsettled. + */ +const UDP_RETESTS = 3; + +/** + * The protocols a pass tests. A node behind UPnP maps one protocol per pass, + * TCP and UDP in turn, so a pass costs its router one mapping and one removal; + * a node with no mapping to make tests both every pass. + * @param {boolean} isUpnp + * @param {object} dosState + * @returns {string[]} + */ +function passProtocols(isUpnp, dosState) { + return isUpnp ? [dosState.testingProtocol ?? 'tcp'] : ['tcp', 'udp']; +} + +/** + * Whether an application the network lists at this node's public address holds + * a port. Another node behind the same address serving that port is not a fault + * of this node's, so the port is not one to test here. + * @param {number} port + * @param {string} localSocketAddress + * @returns {Promise} + */ +async function portHeldAtThisAddress(port, localSocketAddress) { + const appsAtThisAddress = await registryManager.getRunningAppIpList(extractIp(localSocketAddress)); + const held = await portManager.assignedPortsGlobalApps(appsAtThisAddress.map((app) => app.name)); + if (held.some((app) => app.ports.includes(port))) return true; + return (await portManager.siblingHoldingPort([port], localSocketAddress)) !== null; +} + +/** + * What one peer's UDP answer for the port under test says. + * @param {object|undefined} udpAnswered - from the peer + * @param {number} port + * @param {string} token + * @returns {'proven'|'silent'|'unknown'} unknown: the peer does not probe UDP, + * or could not send to the port + */ +function udpVerdict(udpAnswered, port, token) { + if (!udpAnswered || typeof udpAnswered !== 'object') return 'unknown'; + const reply = udpAnswered[port] !== undefined ? udpAnswered[port] : udpAnswered[String(port)]; + if (reply === undefined) return 'unknown'; + return typeof reply === 'string' && reply.includes(token) ? 'proven' : 'silent'; +} + +let wakeNextPass = null; + +/** + * Waits `ms` before the next pass, or less: the router refusing a mapping, a + * UPnP removal becoming pending, or the last one going, starts the next pass at + * once. Every wait between passes is this one, and a wake only ends it: there + * is never more than one pass. + * @param {number} ms + * @returns {Promise} + */ +async function waitForNextPass(ms) { + const woken = new Promise((resolve) => { wakeNextPass = resolve; }); + await Promise.race([serviceHelper.delay(ms), woken]); + wakeNextPass = null; +} + +/** + * Starts the next pass now, if the check is waiting for it. + */ +function wakeCheck() { + if (wakeNextPass) wakeNextPass(); +} + +upnpService.routerEvents.on('mappingRefused', wakeCheck); +upnpService.routerEvents.on('removalPending', wakeCheck); +upnpService.routerEvents.on('removalsClear', wakeCheck); + +/** + * Whether the router has refused a mapping for a reason of its own - a full + * table, no answer - since this check last asked: any mapping, an install's, a + * restore's or this check's own. Read once per pass, so a pass counts once + * however many it refused. + * @param {object} dosState + * @returns {boolean} + */ +function routerRefusedSinceLastPass(dosState) { + const refusals = upnpService.routerRefusals(); + const refused = refusals > dosState.routerRefusalsSeen; + // eslint-disable-next-line no-param-reassign + dosState.routerRefusalsSeen = refusals; + return refused; +} + /** * Check my apps availability by testing ports from external nodes * @param {Function} installedAppsFn - Function to get installed apps @@ -110,7 +230,12 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const { timeoutsMs: timeouts, thresholds } = config.fluxapps.appsPortCheck; const isUpnp = upnpService.isUPNP(); - const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(); + let routerRefused = false; + // One secret for this pass, published by both test servers and never sent to + // the peer: only our own test servers can produce it. + const portTestToken = crypto.randomBytes(16).toString('hex'); + const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(portTestToken); + let udpTestServer = null; const setNextPort = () => { if (dosState.originalPortFailed && dosState.testingPort > dosState.originalPortFailed) { @@ -136,7 +261,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); if (!syncStatus.data.synced) { log.info('Flux Node daemon not synced. Application checks are disabled'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -146,7 +271,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (!isNodeConfirmed) { log.info('Flux Node not Confirmed. Application checks are disabled'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -154,7 +279,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const localSocketAddress = await fluxNetworkHelper.getLocalSocketAddress(); if (!localSocketAddress) { log.info('No Public IP found. Application checks are disabled'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -162,7 +287,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin const installedAppsRes = await installedAppsFn(); if (installedAppsRes.status !== 'success') { log.error('Failed to get installed Apps'); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -204,7 +329,7 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isPortBanned) { log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} is banned`); setNextPort(); - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -214,86 +339,127 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (isPortUpnpBanned) { log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} is UPNP banned`); setNextPort(); - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } } + // A router that has yet to remove a mapping of this node fails the check, + // and is given no test mapping until it has. + if (isUpnp && upnpService.removalPending()) { + log.info('checkMyAppsAvailability - the router has yet to remove a mapping of this node; no test mapping until it has'); + countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); + await waitForNextPass(dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); + setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); + return; + } + if (appPorts.includes(dosState.testingPort)) { log.info(`checkMyAppsAvailability - Skipped checking ${dosState.testingPort} - in use`); setNextPort(); - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } + if (await portHeldAtThisAddress(dosState.testingPort, localSocketAddress)) { + log.info(`checkMyAppsAvailability - Skipped checking ${dosState.testingPort} - held by an application at this address`); + setNextPort(); + await waitForNextPass(timeouts.failure); + setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); + return; + } + + const protocols = passProtocols(isUpnp, dosState); + const tcpTested = protocols.includes('tcp'); + const udpTested = protocols.includes('udp'); + + // A port one peer heard nothing on over UDP is tested again by another. + const udpSuspect = udpTested && dosState.udpSuspect?.port === dosState.testingPort ? dosState.udpSuspect : null; + // An external observer: this asks a peer whether it can reach US, and a Flux // node sharing our public address cannot answer that. Null when there is no // such node, which the retry below already handles. - const remoteSocketAddress = await networkStateService.getRandomExternalObserver(localSocketAddress); + const remoteSocketAddress = await networkStateService.getRandomExternalObserver( + localSocketAddress, + { exclude: udpSuspect ? [udpSuspect.peer] : [] }, + ); if (!remoteSocketAddress) { - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } if (failedNodesTestPortsCache.has(remoteSocketAddress)) { - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } const firewallActive = isArcane ? true : await fluxNetworkHelper.isFirewallActive(); if (firewallActive) { - await fluxNetworkHelper.allowTestPort(dosState.testingPort); + // eslint-disable-next-line no-restricted-syntax + for (const protocol of protocols) { + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.allowTestPort(dosState.testingPort, protocol); + } } if (isUpnp) { - const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App'); - if (!upnpMapResult) { - if (dosState.lastUPNPMapFailed) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); - // eslint-disable-next-line no-param-reassign - dosState.lastUPNPMapFailed = true; - log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} failed to create UPnP mapping`); + // A router refusing either protocol's mapping for a reason of its own is + // a router fault, counted as one. + const upnpMapResult = await upnpService.mapUpnpPort(dosState.testingPort, 'Flux_Test_App', { protocols: protocols.map((protocol) => protocol.toUpperCase()) }); + routerRefused = routerRefusedSinceLastPass(dosState); + if (routerRefused) countPortFailure(dosState, thresholds.dos, UPNP_DOS_MESSAGE); + if (!upnpMapResult.mapped) { + log.info(`checkMyAppsAvailability - Testing port ${dosState.testingPort} failed to create UPnP mapping: ${upnpMapResult.reason}`); setNextPort(); await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, - skipUpnp: true, skipHttpServer: true, + protocols, }); const upnpDelay = dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error; - await serviceHelper.delay(upnpDelay); + await waitForNextPass(upnpDelay); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } - // eslint-disable-next-line no-param-reassign - dosState.lastUPNPMapFailed = false; } - const listening = new Promise((resolve, reject) => { - testHttpServer - .once('error', (err) => { - testHttpServer.removeAllListeners('listening'); - reject(err.message); - }) - .once('listening', () => { - testHttpServer.removeAllListeners('error'); - resolve(null); - }); - testHttpServer.listen(dosState.testingPort); - }); - - const error = await listening.catch((err) => err); + let error = null; + let httpListening = false; + if (tcpTested) { + const listening = new Promise((resolve, reject) => { + testHttpServer + .once('error', (err) => { + testHttpServer.removeAllListeners('listening'); + reject(err.message); + }) + .once('listening', () => { + testHttpServer.removeAllListeners('error'); + resolve(null); + }); + testHttpServer.listen(dosState.testingPort); + }); + error = await listening.catch((err) => err); + httpListening = !error; + } + if (!error && udpTested) { + udpTestServer = new fluxUdpTestServer.FluxUdpTestServer(portTestToken); + error = await udpTestServer.listen(dosState.testingPort).then(() => null, (err) => err.message); + } if (error) { log.warn(`Unable to listen on port: ${dosState.testingPort}. Error: ${error}`); setNextPort(); await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipUpnp: !isUpnp, - skipHttpServer: true, + skipHttpServer: !httpListening, + udpTestServer, + protocols, }); - await serviceHelper.delay(timeouts.error); + await waitForNextPass(timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -318,12 +484,17 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin appname: 'appPortsTest', ports: [dosState.testingPort], pubKey: signer.pubKey, + // Read the port back rather than merely reach it, over this pass's protocols. + echo: true, + protocols, }; const signature = signer.sign(JSON.stringify(data)); if (!signature) throw new Error('checkMyAppsAvailability - the port test could not be signed'); data.signature = signature; + if (udpTestServer) udpTestServer.allow(remoteIp); + const resMyAppAvailability = await axios .post(`http://${remoteIp}:${remotePort}/flux/checkappavailability`, JSON.stringify(data), axiosConfig) .catch(() => { @@ -337,10 +508,13 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipFirewall: !firewallActive, skipUpnp: !isUpnp, + skipHttpServer: !httpListening, + udpTestServer, + protocols, }); if (!resMyAppAvailability) { - await serviceHelper.delay(timeouts.failure); + await waitForNextPass(timeouts.failure); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } @@ -354,13 +528,70 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin if (!['success', 'error'].includes(responseStatus)) { log.warn(`checkMyAppsAvailability - Unexpected response status: ${responseStatus}`); - await serviceHelper.delay(timeouts.error); + await waitForNextPass(timeouts.error); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); return; } - const portTestFailed = responseStatus === 'error'; + const { answered, udpAnswered } = resMyAppAvailability.data.data || {}; + // A pass that does not test TCP reads nothing from the peer's TCP answer: a + // peer that does not know `protocols` tests TCP anyway, and finds it unmapped. + let portTestFailed = tcpTested && responseStatus === 'error'; + let tcpOutcome = 'notTested'; + if (tcpTested) tcpOutcome = portTestFailed ? 'unreachable' : 'passed'; + + // A peer that read the port: only our token says this node is what + // answered. Anything else holding the port leaves it as unusable here as + // nothing answering. A peer that does not read ports is taken at its word. + if (tcpTested && !portTestFailed && answered && portManager.portNotOurs([dosState.testingPort], answered, portTestToken) !== null) { + log.warn(`checkMyAppsAvailability - Port ${dosState.testingPort} answered ${remoteIp}:${remotePort} as something other than this node`); + portTestFailed = true; + tcpOutcome = 'notOurs'; + } + + // Over UDP, our token heard by one peer proves the port; silence heard by + // one peer is tested again by another, and counts once two have heard it. + let retestUdp = false; + let firstUdpSilence = false; + const udp = !udpTested || portTestFailed ? 'unknown' : udpVerdict(udpAnswered, dosState.testingPort, portTestToken); + if (udp === 'proven') { + // eslint-disable-next-line no-param-reassign + dosState.udpSuspect = null; + fluxNetworkHelper.setUdpReachability({ reachable: true }); + } else if (udp === 'silent' && !udpSuspect) { + // The pass's TCP result is handled as any other's; only the port and the + // wait are the retest's. + log.warn(`checkMyAppsAvailability - ${remoteIp}:${remotePort} heard nothing of this node on port ${dosState.testingPort} over UDP; asking another peer`); + // eslint-disable-next-line no-param-reassign + dosState.udpSuspect = { port: dosState.testingPort, peer: remoteSocketAddress, retests: 0 }; + retestUdp = true; + firstUdpSilence = true; + } else if (udp === 'silent') { + const peers = [extractIp(udpSuspect.peer), remoteIp]; + log.error(`checkMyAppsAvailability - Port ${dosState.testingPort} does not answer this node over UDP, as heard by ${peers.join(' and ')}` + + `${udpDosEnabled() ? '' : '; reported in health, not counted'}`); + // eslint-disable-next-line no-param-reassign + dosState.udpSuspect = null; + fluxNetworkHelper.setUdpReachability({ reachable: false, port: dosState.testingPort, peers }); + if (udpDosEnabled()) portTestFailed = true; + } else if (udpSuspect && !portTestFailed) { + udpSuspect.retests += 1; + retestUdp = udpSuspect.retests < UDP_RETESTS; + // eslint-disable-next-line no-param-reassign + if (!retestUdp) dosState.udpSuspect = null; + } else if (udpTested && !tcpTested) { + // A UDP pass whose peer does not probe UDP learned nothing. + // eslint-disable-next-line no-param-reassign + dosState.udpUnsettled = (dosState.udpUnsettled ?? 0) + 1; + retestUdp = dosState.udpUnsettled < UDP_RETESTS; + } + if (udp !== 'unknown' || !retestUdp) { + // eslint-disable-next-line no-param-reassign + dosState.udpUnsettled = 0; + } + let waitMs = 0; + let passed = false; if (portTestFailed && portsNotWorking.size < thresholds.portsHighEdge) { portsNotWorking.add(dosState.testingPort); @@ -391,12 +622,14 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin setRandomPort(); waitMs = timeouts.failure; } else { + passed = true; portsNotWorking.clear(); // eslint-disable-next-line no-param-reassign dosState.nextTestingPort = null; // eslint-disable-next-line no-param-reassign dosState.originalPortFailed = null; - releaseOwnDos(dosState); + // a router refusing others' mappings is not cleared by one that took this test's + if (!routerRefused) releaseOwnDos(dosState); waitMs = timeouts.default; } @@ -410,12 +643,36 @@ async function checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorkin log.error(`checkMyAppsAvailability - Count: ${portsNotWorking.size}. portsNotWorking: ${JSON.stringify(Array.from(portsNotWorking))}`); } - await serviceHelper.delay(waitMs); + if (retestUdp) { + // eslint-disable-next-line no-param-reassign + dosState.nextTestingPort = dosState.testingPort; + waitMs = Math.min(waitMs, timeouts.failure); + } else if (isUpnp && passed) { + // The other protocol's turn, once this one has passed. + // eslint-disable-next-line no-param-reassign + dosState.testingProtocol = tcpTested ? 'udp' : 'tcp'; + } + + // A router that refused this node a mapping or has yet to remove one is + // looked at again soon, so the pass that finds it working again comes + // soon; a refusal during this pass gave no wake, the check not waiting. + const routerFaulting = routerRefused + || upnpService.routerRefusals() > dosState.routerRefusalsSeen + || upnpService.removalPending(); + if (isUpnp && routerFaulting) { + waitMs = Math.min(waitMs, dosState.portFailureScore >= thresholds.dos ? timeouts.dos : timeouts.error); + } + // Which branch each pass took, as a tally: the passes run too often to stream. + fluxEventBus.count('availability:pass', tcpOutcome, udpTested ? udp : 'notTested'); + if (firstUdpSilence) { + fluxEventBus.count('availability:firstUdpSilence', tcpOutcome, portsNotWorking.size === 0 ? 'cleared' : 'notCleared'); + } + await waitForNextPass(waitMs); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } catch (error) { - await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp }); + await handleTestShutdown(dosState.testingPort, testHttpServer, isArcane, { skipUpnp: !isUpnp, udpTestServer, protocols: passProtocols(isUpnp, dosState) }); log.error(`checkMyAppsAvailability - Error: ${error}`); - await serviceHelper.delay(timeouts.appError); + await waitForNextPass(timeouts.appError); setImmediate(() => checkMyAppsAvailability(installedAppsFn, dosState, portsNotWorking, failedNodesTestPortsCache, isArcane)); } } diff --git a/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js b/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js index fde0e7f1e6..df1e9a5f40 100644 --- a/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js +++ b/ZelBack/src/services/appMonitoring/daemonHealthMonitor.js @@ -19,10 +19,10 @@ const REMOVAL_DELAY = 3 * 60 * 1000; // 3 minutes between app removals */ async function removeAllApps(reason) { try { - allAppsRemoved = true; // Set flag to prevent repeated attempts - // Get installed apps (follows pattern from checkAndRemoveEnterpriseAppsOnNonArcane) const installedApps = await registryManager.getInstalledApps(); + // Set once the list is read, so a failed read is retried on the next check. + allAppsRemoved = true; if (!installedApps || installedApps.length === 0) { log.info('No apps installed, nothing to remove'); diff --git a/ZelBack/src/services/appMonitoring/folderStates.js b/ZelBack/src/services/appMonitoring/folderStates.js new file mode 100644 index 0000000000..a8f1c0573b --- /dev/null +++ b/ZelBack/src/services/appMonitoring/folderStates.js @@ -0,0 +1,202 @@ +const globalState = require('../utils/globalState'); +const mountParser = require('../utils/mountParser'); +const dockerService = require('../dockerService'); + +// What this node tells its peers about each receive-only folder of an installed app, +// for a cold-start seed: which holder starts the app from what it holds. +// +// Every such folder has exactly one entry from its install to its removal, so a +// missing entry means only that the folder is not installed here. A peer about to +// seed waits on a folder this node has not looked at, ranks one it has measured, +// leaves out one this node cannot seed, and reads every other as no claim. +// +// Written where each fact becomes known: the install and a volume recreate, the +// monitor pass, a promotion, the removal. Nothing else removes an entry. + +const FolderState = Object.freeze({ + // Installed, and no monitor pass has reached it since. + NOT_LOOKED_AT: 'notLookedAt', + // { bytes, newestModified } of what this node holds that the cluster's index does not. + MEASURED: 'measured', + // Looked at, with nothing to rank: its holdings could not be read, it has sent + // everything it held, or it is a ready standby. + NO_CLAIM: 'noClaim', + // Held out of the monitor pass because its volume is not safe to act on or its app's + // specification cannot be read, with its volume image still on the node: what it + // holds is unknown. + HELD_OUT: 'heldOut', + // Held out of the monitor pass because its volume image is gone: nothing of it is + // on the node. + VOLUME_GONE: 'volumeGone', +}); + +// What a peer is sent for a folder this node has looked at and cannot claim +// anything of. Any value that is not a rankable claim reads as no claim. +const NO_CLAIM_ANSWER = Object.freeze({ unreadable: true }); + +/** + * The folder ids of an app's components that sync receive-only (g: and r:). + * @param {object} appSpec Installed app specification, decrypted. + * @returns {string[]} + */ +function receiveOnlyFolderIds(appSpec) { + const components = appSpec.version <= 3 + ? [{ identifier: appSpec.name, containerData: appSpec.containerData }] + : (appSpec.compose || []).map((component) => ({ identifier: `${component.name}_${appSpec.name}`, containerData: component.containerData })); + return components + .filter(({ containerData }) => ['g', 'r'].includes(mountParser.getComponentSyncMode(containerData))) + .map(({ identifier }) => dockerService.getAppIdentifier(identifier)); +} + +/** + * Records the receive-only folders of an installed app that have no entry as not + * looked at. A folder that already has one keeps it: its volume is the one it + * describes. + * @param {object} appSpec Installed app specification, decrypted. + */ +function installed(appSpec) { + receiveOnlyFolderIds(appSpec) + .filter((folderId) => !globalState.folderStates.has(folderId)) + .forEach((folderId) => globalState.folderStates.set(folderId, { state: FolderState.NOT_LOOKED_AT })); +} + +/** + * Records a folder whose volume has been replaced by an empty one as not looked at. + * @param {string} folderId + */ +function recreated(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.NOT_LOOKED_AT }); +} + +/** + * Removes a folder whose volume has been deleted. + * @param {string} folderId + */ +function removed(folderId) { + globalState.folderStates.delete(folderId); +} + +/** + * Records what a monitor pass measured of a folder. + * @param {string} folderId + * @param {{bytes: number, newestModified: number}|null} holdings null when they could not be read. + */ +function measured(folderId, holdings) { + globalState.folderStates.set(folderId, holdings + ? { state: FolderState.MEASURED, bytes: holdings.bytes, newestModified: holdings.newestModified } + : { state: FolderState.NO_CLAIM }); +} + +/** + * Records a folder the monitor pass held out because its volume is not safe to act on. + * @param {string} folderId + */ +function heldOut(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.HELD_OUT }); +} + +/** + * Records a folder the monitor pass held out because its volume image is gone. + * @param {string} folderId + */ +function volumeGone(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.VOLUME_GONE }); +} + +function heldOutUnlessGone(folderId) { + if (globalState.folderStates.get(folderId)?.state === FolderState.VOLUME_GONE) return; + heldOut(folderId); +} + +/** + * Records a folder the monitor pass found not mounted without asking why. One whose + * volume image is known to be gone stays so; any other is held out. + * @param {string} folderId + */ +function notMounted(folderId) { + heldOutUnlessGone(folderId); +} + +/** + * Records a folder of an app whose specification this node cannot read, which the + * monitor pass does not reach. One whose volume image is known to be gone stays so; + * any other is held out. + * @param {string} folderId + */ +function specUnreadable(folderId) { + heldOutUnlessGone(folderId); +} + +/** + * Records a folder that has turned writable: everything this node held of it is + * published, so it has nothing left to claim. + * @param {string} folderId + */ +function sending(folderId) { + globalState.folderStates.set(folderId, { state: FolderState.NO_CLAIM }); +} + +/** + * Records a folder the monitor pass found a ready standby. A measurement it holds + * stands; any other state becomes no claim, because a ready standby is not measured. + * @param {string} folderId + */ +function settled(folderId) { + if (globalState.folderStates.get(folderId)?.state === FolderState.MEASURED) return; + globalState.folderStates.set(folderId, { state: FolderState.NO_CLAIM }); +} + +/** + * What this node claims of a folder, as a peer reads it: a rankable claim, the no-claim + * answer, or undefined for a folder it has not looked at or does not hold. + * @param {string} folderId + * @returns {{bytes: number, newestModified: number}|object|undefined} + */ +function claimOf(folderId) { + const entry = globalState.folderStates.get(folderId); + if (!entry || entry.state === FolderState.NOT_LOOKED_AT) return undefined; + if (entry.state === FolderState.MEASURED) return { bytes: entry.bytes, newestModified: entry.newestModified }; + return NO_CLAIM_ANSWER; +} + +/** + * The receive-only part of the promoted-folders answer. A writable folder is in + * none of its lists: a peer reads it from the folders this node sends. Every other + * folder is in exactly one. + * @param {Set} writable The folders this node holds writable. + * @returns {{holding: object, notLookedAt: string[], heldOut: string[], volumeGone: string[]}} + */ +function answer(writable) { + const holding = {}; + const notLookedAt = []; + const heldOutIds = []; + const volumeGoneIds = []; + globalState.folderStates.forEach((entry, folderId) => { + if (writable.has(folderId)) return; + if (entry.state === FolderState.NOT_LOOKED_AT) notLookedAt.push(folderId); + else if (entry.state === FolderState.HELD_OUT) heldOutIds.push(folderId); + else if (entry.state === FolderState.VOLUME_GONE) volumeGoneIds.push(folderId); + else holding[folderId] = claimOf(folderId); + }); + return { + holding, notLookedAt, heldOut: heldOutIds, volumeGone: volumeGoneIds, + }; +} + +module.exports = { + FolderState, + NO_CLAIM_ANSWER, + answer, + claimOf, + heldOut, + installed, + measured, + notMounted, + receiveOnlyFolderIds, + recreated, + removed, + sending, + settled, + specUnreadable, + volumeGone, +}; diff --git a/ZelBack/src/services/appMonitoring/localChangeCheck.js b/ZelBack/src/services/appMonitoring/localChangeCheck.js new file mode 100644 index 0000000000..9a22a73e00 --- /dev/null +++ b/ZelBack/src/services/appMonitoring/localChangeCheck.js @@ -0,0 +1,300 @@ +const crypto = require('crypto'); +const log = require('../../lib/log'); +const syncthingService = require('../syncthingService'); +const messageHelper = require('../messageHelper'); +const fluxEventBus = require('../utils/fluxEventBus'); +const syncthingFolderWrites = require('./syncthingFolderWrites'); + +// Whether a receive-only folder's local changes include the only copy of data another +// device wrote, decided off the monitor pass. +// +// The answer takes a request per change to another device's data, and an app decides +// how many of those there are: a standby that rewrote its copy of a large folder has +// one for every file. Asked inside the pass, that is paid by every app on the node. +// Here a check runs on its own, one folder at a time per node, and the pass reads the +// answer it left. +// +// AN ANSWER HOLDS FOR THE FOLDER IT WAS ASKED ABOUT, AND NO OTHER. It is stamped with +// the folder as syncthing described it when the check was asked for: the counts of +// this node's changes to it, and the devices it is shared with. A change on this node +// moves the first, and a device leaving the folder - the moment a change held +// elsewhere becomes the only copy - changes the second. What this node pulls moves +// neither, so an answer stands while the primary keeps writing. The pass acts on an +// answer only while the folder still carries its stamp. +// +// A REVERT IS OF THE CHANGES CHECKED, AND NO OTHERS, ON THE FOLDER CHECKED. Counts can +// stay the same while the changes do not, so the pass does not revert itself: it asks +// for a revert here, in the same queue as the checks. The revert reads the folder's +// changes again and goes ahead only while they are exactly the ones the check +// examined, and the folder still carries the check's stamp. The stamp is read and the +// revert sent while the folder's config is held against every write +// (syncthingFolderWrites), so no device leaves the folder between the two; a write +// that arrives once syncthing has the revert waits inside syncthing for its work. + +// syncthing's FlagLocalReceiveOnly: an item this node changed in a receive-only folder. +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const LOCAL_CHANGE_PAGE_SIZE = 1000; +// How many items are asked about at once. +const FILE_REQUESTS_AT_ONCE = 8; + +// The counts syncthing keeps of a folder's receive-only changes, which make its stamp. +const CHANGE_COUNTS = Object.freeze([ + 'receiveOnlyChangedFiles', 'receiveOnlyChangedDirectories', 'receiveOnlyChangedSymlinks', + 'receiveOnlyChangedDeletes', 'receiveOnlyChangedBytes', +]); + +// Jobs asked for and not started, at most one per folder: a check +// { kind: 'check', folderId, entry, stamp, isCurrent } or a revert +// { kind: 'revert', folderId, entry, answer, isCurrent, files, directories }. +const waiting = []; +let running = null; + +/** + * The folder as syncthing describes it, in the form an answer is stamped with. + * @param {object} status The folder's db/status: its counts of receive-only changes, + * each absent one read as none. + * @param {Array<{deviceID: string}>} devices The devices the folder is shared with. + * @returns {string} + */ +function stampOf(status, devices) { + const changes = CHANGE_COUNTS.map((count) => status?.[count] ?? 0); + const shared = devices.map((device) => device.deviceID).sort(); + return JSON.stringify({ changes, shared }); +} + +/** + * Adds one listed change to a fingerprint of the folder's changes: its name, its + * history and whether it is a deletion. + * @param {crypto.Hash} hash + * @param {object} item An entry of db/localchanged. + */ +function addToFingerprint(hash, item) { + hash.update(`${item?.name}\0${(item?.version ?? []).join(',')}\0${item?.deleted ? 1 : 0}\n`); +} + +/** + * A fingerprint of every local change of a folder, as syncthing lists them now. + * @param {string} folderId + * @returns {Promise} null when the changes could not be read + */ +async function changesFingerprint(folderId) { + try { + const hash = crypto.createHash('sha256'); + for (let page = 1; ; page += 1) { + // eslint-disable-next-line no-await-in-loop + const batch = (await syncthingService.getDbLocalChanged(folderId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; + if (!Array.isArray(batch)) return page > 1 ? hash.digest('hex') : null; + batch.forEach((item) => addToFingerprint(hash, item)); + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return hash.digest('hex'); + } + } catch (error) { + log.warn(`localChangeCheck - ${folderId}: ${error.message}`); + return null; + } +} + +/** + * The first of the named items whose cluster version is this node's own change: no + * other device holds a valid version of it. + * @param {string} folderId + * @param {string[]} names + * @returns {Promise} + */ +async function firstHeldOnlyHere(folderId, names) { + for (let start = 0; start < names.length; start += FILE_REQUESTS_AT_ONCE) { + const batch = names.slice(start, start + FILE_REQUESTS_AT_ONCE); + // eslint-disable-next-line no-await-in-loop + const answers = await Promise.all(batch.map((name) => syncthingService.getDbFile(folderId, name))); + // eslint-disable-next-line no-bitwise + const index = answers.findIndex((answer) => ((answer?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0); + if (index >= 0) return batch[index]; + } + return undefined; +} + +/** + * What a receive-only folder's local changes are, for whether they may be reverted: + * - `onlyCopy`: a change that is the only copy of data from another device. Its + * history names another device, so the data came from elsewhere, and no other + * device holds a valid version of it. syncthing reverts such an item by deleting + * it, and a directory that is not empty cannot be deleted, so that revert fails on + * every pass. The first one found ends the check. + * - `madeHere`: whether any change's history is this node's alone. Such a change was + * made on this node, as a standby or earlier as the primary, and no other device's + * history says otherwise. It is not asked about: whether it may be reverted turns + * on whether another holder runs the app, not on what other devices hold. + * - `changes`: a fingerprint of every change examined, when there is no only copy; + * a revert goes ahead only while the folder's changes still have it. + * @param {string} folderId + * @returns {Promise<{onlyCopy: string|false, madeHere: boolean, changes?: string}|null>} + * null when the changes could not be read + */ +async function examineLocalChanges(folderId) { + try { + const deviceId = await syncthingService.getDeviceId(); + if (!deviceId) return null; + const ownCounter = `${deviceId.substring(0, 7)}:`; + const hash = crypto.createHash('sha256'); + let madeHere = false; + for (let page = 1; ; page += 1) { + // eslint-disable-next-line no-await-in-loop + const batch = (await syncthingService.getDbLocalChanged(folderId, page, LOCAL_CHANGE_PAGE_SIZE))?.files; + if (!Array.isArray(batch)) return page > 1 ? { onlyCopy: false, madeHere, changes: hash.digest('hex') } : null; + const fromElsewhere = []; + batch.forEach((item) => { + addToFingerprint(hash, item); + if ((item?.version ?? []).some((counter) => !counter.startsWith(ownCounter))) fromElsewhere.push(item.name); + else madeHere = true; + }); + // eslint-disable-next-line no-await-in-loop + const found = await firstHeldOnlyHere(folderId, fromElsewhere); + if (found !== undefined) return { onlyCopy: found, madeHere }; + if (batch.length < LOCAL_CHANGE_PAGE_SIZE) return { onlyCopy: false, madeHere, changes: hash.digest('hex') }; + } + } catch (error) { + log.warn(`localChangeCheck - ${folderId}: ${error.message}`); + return null; + } +} + +/** + * Runs one check and leaves its answer on the folder's entry, unless the entry has been + * replaced since it was asked for: the folder it describes is gone. + * @param {{folderId: string, entry: object, stamp: string, isCurrent: Function}} check + */ +async function runCheck(check) { + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_CHECK, check.folderId); + const found = await examineLocalChanges(check.folderId); + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER, check.folderId); + if (found === null || !check.isCurrent()) return; + // eslint-disable-next-line no-param-reassign + check.entry.localChangeCheck = { stamp: check.stamp, ...found }; + fluxEventBus.publish('syncthing:localChangesChecked', { folder: check.folderId, onlyCopy: Boolean(found.onlyCopy), madeHere: found.madeHere }); +} + +/** + * The folder as syncthing describes it now, in the form an answer is stamped with. + * @param {string} folderId + * @returns {Promise} null when it could not be read + */ +async function currentStamp(folderId) { + try { + const [status, folder] = await Promise.all([syncthingService.getDbStatus(folderId), syncthingService.getConfigFolders(folderId)]); + return Array.isArray(folder?.devices) ? stampOf(status, folder.devices) : null; + } catch (error) { + log.warn(`localChangeCheck - ${folderId}: ${error.message}`); + return null; + } +} + +/** + * Reverts a folder's local changes if they are exactly the ones its answer examined, + * the folder still carries the answer's stamp, and the entry has not been replaced. + * Otherwise the answer is dropped, so the pass asks for the folder to be checked + * again; a folder that could not be read keeps it. + * @param {{folderId: string, entry: object, answer: object, isCurrent: Function, files: number, directories: number}} job + */ +async function runRevert(job) { + const { folderId, entry, answer } = job; + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT, folderId); + const now = await changesFingerprint(folderId); + if (now === null || !job.isCurrent() || entry.localChangeCheck !== answer) return; + if (now !== answer.changes) { + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + log.info(`localChangeCheck - ${folderId}: its local changes are not the ones checked; checking them again before any revert`); + return; + } + await syncthingFolderWrites.exclusive([folderId], async () => { + const stamp = await currentStamp(folderId); + if (stamp === null || !job.isCurrent() || entry.localChangeCheck !== answer) return; + if (stamp !== answer.stamp) { + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + log.info(`localChangeCheck - ${folderId}: the folder is not the one checked; checking it again before any revert`); + return; + } + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT_SENT, folderId); + log.warn(`localChangeCheck - ${folderId} is synced but the receive-only folder has local changes (${job.files} files, ${job.directories} directories); reverting to the cluster's version`); + try { + // dataOrThrow: dbRevert answers in-band; without it a failed revert reads as reverted + messageHelper.dataOrThrow(await syncthingService.dbRevert(folderId)); + // eslint-disable-next-line no-param-reassign + delete entry.localChangeCheck; + fluxEventBus.publish('syncthing:localChangesReverted', { folder: folderId, files: job.files, directories: job.directories }); + } catch (error) { + log.error(`localChangeCheck - revert of local changes for ${folderId} failed: ${error.message}`); + } + }); +} + +async function runWaitingJobs() { + while (waiting.length) { + running = waiting.shift(); + // eslint-disable-next-line no-await-in-loop + await (running.kind === 'revert' ? runRevert(running) : runCheck(running)); + } + running = null; +} + +// The waiting job for `job`'s folder becomes `job`. +function enqueue(job) { + const index = waiting.findIndex((queued) => queued.folderId === job.folderId); + if (index >= 0) waiting[index] = job; + else waiting.push(job); + if (!running) runWaitingJobs(); +} + +/** + * Asks for a folder's local changes to be checked against the folder as `stamp` + * describes it. A job already asked for the folder is replaced; a check running for + * the same folder, entry and stamp is left to finish. + * @param {string} folderId + * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache; the answer + * is left on it as `localChangeCheck`. + * @param {string} stamp stampOf the folder now. + * @param {Function} isCurrent Whether `entry` is still the folder's entry. + */ +function request(folderId, entry, stamp, isCurrent) { + if (running?.kind === 'check' && running.folderId === folderId && running.entry === entry && running.stamp === stamp) return; + enqueue({ + kind: 'check', folderId, entry, stamp, isCurrent, + }); +} + +/** + * Asks for a folder's local changes to be reverted, if they are still the ones + * `answer` examined. A job already asked for the folder is replaced; a revert running + * for the same answer is left to finish. + * @param {string} folderId + * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache. + * @param {object} answer The answer the revert rests on, left on `entry`. + * @param {Function} isCurrent Whether `entry` is still the folder's entry. + * @param {{files: number, directories: number}} changed What the folder's status counts. + */ +function revert(folderId, entry, answer, isCurrent, { files, directories }) { + if (running?.kind === 'revert' && running.folderId === folderId && running.answer === answer) return; + enqueue({ + kind: 'revert', folderId, entry, answer, isCurrent, files, directories, + }); +} + +/** + * The answer left for a folder's entry, if it was made against the folder as it is now. + * @param {object} entry The folder's entry in receiveOnlySyncthingAppsCache. + * @param {string} stamp stampOf the folder now. + * @returns {{onlyCopy: string|false, madeHere: boolean}|null} + */ +function answerFor(entry, stamp) { + const answer = entry?.localChangeCheck; + return answer && answer.stamp === stamp ? answer : null; +} + +module.exports = { + answerFor, + changesFingerprint, + examineLocalChanges, + request, + revert, + stampOf, +}; diff --git a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js index 49c6bebdb0..99ebb8f3b4 100644 --- a/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js +++ b/ZelBack/src/services/appMonitoring/nodeStatusMonitor.js @@ -8,10 +8,46 @@ const fluxCommunicationUtils = require('../fluxCommunicationUtils'); const messageStore = require('../appMessaging/messageStore'); const nodeConfirmationService = require('../nodeConfirmationService'); const networkStateService = require('../networkStateService'); +const hostStallService = require('../hostStallService'); +const appKill = require('../appLifecycle/appKill'); +const dockerService = require('../dockerService'); +const { RemovalOutcome } = require('../utils/removalOutcome'); +const { withinDeadline, TIMED_OUT } = require('../utils/withinDeadline'); +const fluxEventBus = require('../utils/fluxEventBus'); +const globalState = require('../utils/globalState'); const log = require('../../lib/log'); const { extractIp, extractPort } = require('../utils/socketAddressUtils'); let removalInProgress = false; +let unloadingForHostStall = false; +/** What initialize was given, for the removal a host stall starts. */ +let deps = null; + +const REMOVAL_DEADLINE_MS = config.fluxapps.appRemovalDeadlineMs; +const STOP_WINDOW_MS = config.hostStall.stopWindowMs; +const STILL_REMOVING = 'stillRemoving'; +/** Apps whose removal started here and has not returned, deadline or not. */ +const removalsRunning = new Set(); + +/** + * Remove one app, waiting for it at most REMOVAL_DEADLINE_MS. A removal past + * its deadline keeps running, so an app whose removal is still running is not + * removed again: a second removal beside it would only wait behind the same + * stall, and the next pass would add a third. Boot removes through this too, so a + * removal boot stopped waiting for is one the monitor does not start again. + * @param {Function} removeAppLocallyFn + * @param {string} name + * @param {boolean} canBroadcast + * @returns {Promise} The removal's outcome, TIMED_OUT, or STILL_REMOVING. + */ +async function removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) { + if (removalsRunning.has(name)) return STILL_REMOVING; + removalsRunning.add(name); + const removal = Promise.resolve() + .then(() => removeAppLocallyFn(name, null, true, false, canBroadcast)) + .finally(() => removalsRunning.delete(name)); + return withinDeadline(removal, REMOVAL_DEADLINE_MS); +} // Database collections const globalAppsLocations = config.database.appsglobal.collections.appsLocations; @@ -34,11 +70,20 @@ async function removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason) const appsInstalled = installedAppsRes.data; const canBroadcast = nodeConfirmationService.canSendMessages(); for (const installedApp of appsInstalled) { + // a host stall's own removal has the node, and this one gives way to it + if (unloadingForHostStall) break; log.info(`monitorNodeStatus - Application ${installedApp.name} going to be removed: ${reason}`); log.warn(`monitorNodeStatus - Removing application ${installedApp.name} locally`); + // One removal that never returns must not hold every other app here. // eslint-disable-next-line no-await-in-loop - await removeAppLocallyFn(installedApp.name, null, true, false, canBroadcast); - log.warn(`monitorNodeStatus - Application ${installedApp.name} locally removed`); + const outcome = await removeWithinDeadline(removeAppLocallyFn, installedApp.name, canBroadcast); + if (outcome === TIMED_OUT) { + log.error(`monitorNodeStatus - Application ${installedApp.name} was not removed within ${REMOVAL_DEADLINE_MS / 1000}s, moving on`); + } else if (outcome === STILL_REMOVING) { + log.warn(`monitorNodeStatus - Application ${installedApp.name} is still being removed, moving on`); + } else { + log.warn(`monitorNodeStatus - Application ${installedApp.name} locally removed`); + } // eslint-disable-next-line no-await-in-loop await serviceHelper.delay(config.fluxapps.nodeMonitorRemovalDelayMs ?? 60000); } @@ -47,16 +92,153 @@ async function removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason) } } +/** + * Removal started from a listener: nothing awaits it, so its failure is logged + * here. The periodic monitor tries again on its next pass. + * @param {Function} installedAppsFn + * @param {Function} removeAppLocallyFn + * @param {string} reason + * @returns {Promise} + */ +async function removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, reason) { + try { + await removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, reason); + } catch (error) { + log.error(`monitorNodeStatus - app removal (${reason}) failed: ${error.message}`); + } +} + +/** + * Ask docker to stop every container of the app, on its normal grace. + * @param {string} name + * @param {Array|null} containers appKill.readContainers + * @returns {Promise} + */ +async function stopApp(name, containers) { + await Promise.allSettled(appKill.containersOf(name, containers).map((container) => dockerService.appDockerStop(container))); +} + +/** + * Remove every app from a node whose host is stalled, and make sure none can + * run again before the network is told. + * + * Every app is asked to stop at once, and all of them share one STOP_WINDOW_MS: + * a docker that can still deliver a stop signal stops each app cleanly, and one + * that cannot costs the window once, not once per app. Whatever of each app can + * still run after the window is killed through its cgroups and checked: an app + * whose every process is gone or has the kill pending can never run its code + * again. Then each app gets its normal removal, which tells the network, within + * REMOVAL_DEADLINE_MS; nothing of it runs by then, so the removals take turns. + * + * The apps are read from the database and from what is on disk, at once, so an + * app the database cannot list is still found. Docker's records of its + * containers are read once, within STOP_WINDOW_MS: on a disk that does not answer + * they are not waited for further, and every app is still stopped and killed + * through its slice, with every container outside the slices (appKill). An app + * is reported with runnable null only when systemd does not say which containers + * are outside the slices. + * @param {Function} installedAppsFn + * @param {Function} removeAppLocallyFn + */ +async function unloadForHostStall(installedAppsFn, removeAppLocallyFn) { + if (unloadingForHostStall) return; + unloadingForHostStall = true; + try { + // Read at once: a stall that holds one of them is likely to hold the other. + const [listed, read] = await Promise.all([ + withinDeadline(installedAppsFn(), REMOVAL_DEADLINE_MS).catch(() => null), + withinDeadline(appKill.readContainers(), STOP_WINDOW_MS), + ]); + const installed = listed?.status === 'success' ? listed.data.map((app) => app.name) : []; + const containers = Array.isArray(read) ? read : null; + if (!containers) log.error(`monitorNodeStatus - host stall: docker's container records could not be read (${read === TIMED_OUT ? 'no answer' : 'refused'}); every container outside the app slices is killed`); + const names = [...new Set([...installed, ...await appKill.appsOnDisk(containers)])]; + log.warn(`monitorNodeStatus - host stall: stopping ${names.length} applications within one ${STOP_WINDOW_MS / 1000}s window`); + await withinDeadline(Promise.allSettled(names.map((name) => stopApp(name, containers))), STOP_WINDOW_MS); + + // With no records, a container outside the slices cannot be named, whatever apps + // were found: every one is killed and checked, once. + let outsideSlices; + if (!containers) { + const killedIn = await appKill.killOutsideSlices(); + const runnable = await appKill.stillRunnableOutsideSlices(); + outsideSlices = { containers: killedIn?.length ?? null, runnable }; + if (runnable === null) log.error('monitorNodeStatus - host stall: which containers are outside the app slices cannot be known'); + else if (runnable.length) log.error(`monitorNodeStatus - host stall: processes ${runnable.join(', ')} of containers outside the app slices can still run`); + else log.warn(`monitorNodeStatus - host stall: nothing of the ${killedIn.length} containers outside the app slices can run`); + } + + const stopped = new Map(); + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + // eslint-disable-next-line no-await-in-loop + let runnable = await appKill.stillRunnable(name, containers); + const killed = Boolean(runnable?.length); + if (killed) { + log.error(`monitorNodeStatus - Application ${name} still runs after being asked to stop, killing it`); + // eslint-disable-next-line no-await-in-loop + await appKill.kill(name, containers); + // eslint-disable-next-line no-await-in-loop + runnable = await appKill.stillRunnable(name, containers); + if (runnable?.length) { + log.error(`monitorNodeStatus - Application ${name} could not be stopped: processes ${runnable.join(', ')} can still run`); + } else if (runnable) { + log.warn(`monitorNodeStatus - Application ${name} can no longer run`); + } + } + if (runnable === null) log.error(`monitorNodeStatus - Application ${name}: whether it can still run cannot be known`); + stopped.set(name, { killed, runnable }); + } + // runnable: the processes of each app that can still run its code, empty when + // none can, null when that cannot be known; outsideSlices, when the records could + // not be read: how many containers outside the slices were killed, and which of + // their processes can still run + fluxEventBus.publish('hostStall:appsStopped', { + apps: names.map((name) => ({ app: name, ...stopped.get(name) })), + ...(outsideSlices ? { outsideSlices } : {}), + }); + + const canBroadcast = nodeConfirmationService.canSendMessages(); + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + log.warn(`monitorNodeStatus - Application ${name} going to be removed: host stall`); + // eslint-disable-next-line no-await-in-loop + const outcome = await removeWithinDeadline(removeAppLocallyFn, name, canBroadcast) + .catch(() => RemovalOutcome.FAILED); + const { killed, runnable } = stopped.get(name); + if (runnable?.length === 0 && outcome !== RemovalOutcome.REMOVED && outcome !== RemovalOutcome.NOT_INSTALLED) { + log.warn(`monitorNodeStatus - Application ${name} was not removed (${outcome}), and can no longer run`); + } + fluxEventBus.publish('hostStall:appUnloaded', { + app: name, removal: outcome, killed, runnable, + }); + } + } finally { + unloadingForHostStall = false; + } +} + +/** The removal a host stall starts. Nothing awaits it, so its failure is logged here. */ +async function unloadOnHostStall() { + try { + await unloadForHostStall(deps.installedAppsFn, deps.removeAppLocallyFn); + } catch (error) { + log.error(`monitorNodeStatus - app removal (host stall) failed: ${error.message}`); + } +} + function initialize(installedAppsFn, removeAppLocallyFn) { + deps = { installedAppsFn, removeAppLocallyFn }; + hostStallService.onTrip(unloadOnHostStall); nodeConfirmationService.onConfirmationChange((confirmed) => { if (!confirmed) { log.info('monitorNodeStatus - Confirmation lost, triggering immediate app removal'); - removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, 'node lost confirmation'); + removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, 'node lost confirmation'); } }); nodeConfirmationService.onDaemonStale(() => { log.info('monitorNodeStatus - Daemon stale, triggering app removal'); - removeAllAppsLocally(installedAppsFn, removeAppLocallyFn, 'daemon unreachable'); + removeAllAppsOnSignal(installedAppsFn, removeAppLocallyFn, 'daemon unreachable'); }); } @@ -157,7 +339,22 @@ async function monitorNodeStatus(installedAppsFn, removeAppLocallyFn) { } } +/** + * Start the monitor once boot has settled, whatever boot decided: boot removes + * every app itself when the node may run none, and the two never remove at once. + * @param {Function} installedAppsFn + * @param {Function} removeAppLocallyFn + */ +async function start(installedAppsFn, removeAppLocallyFn) { + await globalState.waitForBootContainerStateSettled(); + monitorNodeStatus(installedAppsFn, removeAppLocallyFn); +} + module.exports = { initialize, + start, monitorNodeStatus, + removeWithinDeadline, + unloadForHostStall, + STILL_REMOVING, }; diff --git a/ZelBack/src/services/appMonitoring/peerComponent.js b/ZelBack/src/services/appMonitoring/peerComponent.js index 01cad8e0b3..cd9d571bb9 100644 --- a/ZelBack/src/services/appMonitoring/peerComponent.js +++ b/ZelBack/src/services/appMonitoring/peerComponent.js @@ -8,8 +8,8 @@ const { const { silenceVerdict, SilenceVerdict } = require('./peerFolderLiveness'); /** - * What this node can show about a peer's copy of a g: component. UNKNOWN is not - * a soft NOT_RUNNING: only NOT_RUNNING releases the component for a start here, + * What this node can show about a peer's copy of an app's g: components. UNKNOWN + * is not a soft NOT_RUNNING: only NOT_RUNNING releases them for a start here, * because starting is what puts a second writer on a shared volume. */ const PeerComponent = Object.freeze({ @@ -33,7 +33,11 @@ const SILENCE_REASONS = Object.freeze({ }); /** - * What this node can show one peer to be doing with a component. + * What this node can show one peer to be doing with an app's g: components. + * + * The peer holds the app when it holds ANY of them. Its primary runs them all on + * one node, so a peer holding one has the others' writes too, and starting one + * beside it splits the app across two nodes. * * The election, the surplus rule and a returning primary's stand-down ask this * question through the same code. Two implementations of "is that peer running @@ -45,7 +49,7 @@ const SILENCE_REASONS = Object.freeze({ * primary, or from the instance order the surplus rule ranks. * @param {string} peerSocketAddr The peer's socket address. * @param {object} ctx Everything the probe needs that is not the peer. - * @param {string} ctx.appId Container name for the component. + * @param {string[]} ctx.appIds Container names of the app's g: components. * @param {string} ctx.identifier `_` the election keys on. * @param {string} ctx.appName Global app name, for the log lines. * @param {object} ctx.liveness Peer folder liveness, for judging silence. @@ -54,16 +58,17 @@ const SILENCE_REASONS = Object.freeze({ * @returns {Promise} A PeerComponent state. */ async function peerComponentState(peerSocketAddr, { - appId, identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }) { // Docker reports names with a leading slash, and getAppIdentifier yields - // exactly the container name for this component. Compare whole names: a + // exactly the container name for each component. Compare whole names: a // substring test also matches a longer app whose name merely begins the same // way - myapp against myapp2, or simplexsmp against simplexsmp1 - and a false - // positive here means the component is never started at all. - const peerRunsThisComponent = (appsRunning) => appsRunning.some( - (app) => (app.Names || []).some((name) => name.replace(/^\//, '') === appId), - ); + // positive here means the app is never started at all. + const holdsAny = (names) => appIds.some((appId) => names.includes(appId)); + const peerRunsThisComponent = (appsRunning) => holdsAny(appsRunning.flatMap( + (app) => (app.Names || []).map((name) => name.replace(/^\//, '')), + )); const ipToCheck = extractIp(peerSocketAddr); const portToCheck = extractPort(peerSocketAddr); @@ -71,7 +76,7 @@ async function peerComponentState(peerSocketAddr, { const { IdentityVerdict, AnswerPurpose } = peerIdentityService; const silent = () => silentPeerState(peerSocketAddr, { - appId, identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }); // "Not running" is a clearance to start a second writer, so it counts only @@ -97,9 +102,9 @@ async function peerComponentState(peerSocketAddr, { log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} signed an answer that does not list what it holds for app:${appName}, will not start`); return PeerComponent.UNKNOWN; } - if (held.includes(appId)) { + if (holdsAny(held)) { fluxEventBus.count('masterSlave:decision', identifier, 'heldOnPeer'); - log.info(`${logPrefix}: component:${identifier} is held on peer node (${label}) at ${ipToCheck}, will not start`); + log.info(`${logPrefix}: app:${appName} is held on peer node (${label}) at ${ipToCheck}, will not start`); return PeerComponent.RUNNING; } return PeerComponent.NOT_RUNNING; @@ -134,9 +139,9 @@ async function peerComponentState(peerSocketAddr, { }); const held = heldResponse?.data?.data; if (Array.isArray(held)) { - if (held.includes(appId)) { + if (holdsAny(held)) { fluxEventBus.count('masterSlave:decision', identifier, 'heldOnPeer'); - log.info(`${logPrefix}: component:${identifier} is held on peer node (${label}) at ${ipToCheck}, will not start`); + log.info(`${logPrefix}: app:${appName} is held on peer node (${label}) at ${ipToCheck}, will not start`); return PeerComponent.RUNNING; } return PeerComponent.NOT_RUNNING; @@ -165,11 +170,11 @@ async function peerComponentState(peerSocketAddr, { log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is alive but did not list what it runs for app:${appName}, will not start`); return PeerComponent.UNKNOWN; } - // Match on the g: component identifier, not the app name: non-g siblings + // Match on the g: component identifiers, not the app name: non-g siblings // (e.g. a DB cluster component) run on every node and must not be mistaken - // for the master/slave component being active there. + // for the app's primary being there. if (peerRunsThisComponent(appsRunning)) { - log.info(`${logPrefix}: component:${identifier} is running on peer node (${label}) at ${ipToCheck}, will not start`); + log.info(`${logPrefix}: app:${appName} is running on peer node (${label}) at ${ipToCheck}, will not start`); return PeerComponent.RUNNING; } return PeerComponent.NOT_RUNNING; @@ -185,21 +190,23 @@ async function peerComponentState(peerSocketAddr, { } /** - * What a peer that did not reply at all can be shown to be doing with a - * component: free only when this node's own syncthing shows the peer's - * connection to the folder gone. + * What a peer that did not reply at all can be shown to be doing with an app's + * g: components: free only when this node's own syncthing shows the peer's + * connection gone for every one of their folders. A folder still connected is a + * component the peer may still be writing. * @param {string} peerSocketAddr The peer's socket address. * @param {object} ctx As peerComponentState. * @returns {Promise} NOT_RUNNING or UNKNOWN. */ async function silentPeerState(peerSocketAddr, { - appId, identifier, appName, liveness, label, logPrefix, + appIds, identifier, appName, liveness, label, logPrefix, }) { const ipToCheck = extractIp(peerSocketAddr); - const verdict = await silenceVerdict(appId, peerSocketAddr, liveness); + const verdicts = await Promise.all(appIds.map((appId) => silenceVerdict(appId, peerSocketAddr, liveness))); + const verdict = verdicts.find((each) => each !== SilenceVerdict.GONE) ?? SilenceVerdict.GONE; fluxEventBus.count('peer:silenceVerdict', identifier, verdict); if (verdict === SilenceVerdict.GONE) { - log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appId} gone - the component is free there`); + log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent and this node's syncthing shows its connection for ${appIds.join(', ')} gone - the app is free there`); return PeerComponent.NOT_RUNNING; } log.info(`${logPrefix}: peer node (${label}) at ${ipToCheck} is silent for app:${appName} and ${SILENCE_REASONS[verdict]}, will not start`); @@ -208,10 +215,10 @@ async function silentPeerState(peerSocketAddr, { /** * Whether a peer is shown, in its own signed words, not to have decided it holds - * a component: it neither runs it, nor is committed to or becoming its primary, - * nor was stopped there by its owner as its primary. It holds it at most by a - * sending folder - a primary back from a stop, still deciding whether another - * holder took over, which gives way to one that has. + * an app: of its g: components it runs none, is committed to or becoming the + * primary of none, and was stopped there by its owner as the primary of none. It + * holds them at most by sending folders - a primary back from a stop, still + * deciding whether another holder took over, which gives way to one that has. * * Anything short of a signed answer saying so is false: an unsigned or unproven * reply, a peer that cannot be reached, or one too old to say what it decided. @@ -220,7 +227,7 @@ async function silentPeerState(peerSocketAddr, { * @returns {Promise} */ async function peerUndecided(peerSocketAddr, { - appId, identifier, appName, label, logPrefix, + appIds, identifier, appName, label, logPrefix, }) { const { IdentityVerdict, AnswerPurpose } = peerIdentityService; const asked = await peerIdentityService.askSigned( @@ -235,14 +242,14 @@ async function peerUndecided(peerSocketAddr, { log.info(`${logPrefix}: peer node (${label}) at ${extractIp(peerSocketAddr)} cannot be shown to be undecided about app:${appName}`); return false; } - const undecided = !decided.includes(appId); + const undecided = !appIds.some((appId) => decided.includes(appId)); if (undecided) fluxEventBus.count('masterSlave:decision', identifier, 'peerUndecided'); return undecided; } /** - * What this node can show the other holders of a component to be doing with it, - * taken together. One peer that cannot be ruled out decides on its own: every + * What this node can show the other holders of an app to be doing with its g: + * components, taken together. One peer that cannot be ruled out decides on its own: every * other peer answering "not me" says nothing about that one. * @param {Array<{ip: string, label: string}>} peers The other holders. * @param {object} ctx As peerComponentState, without `label`. @@ -259,9 +266,10 @@ async function componentStateOnPeers(peers, ctx) { } /** - * What every other holder of a component is doing with it, read from its app - * locations: the one question a primary returning paused asks before its unsent - * changes go out or are discarded, whichever pass reaches the folder first. + * What every other holder of an app is doing with its g: components, read from + * its app locations: the one question a primary returning paused asks before its + * unsent changes go out or are discarded, whichever pass reaches the folder + * first. * @param {(appName: string) => Promise>} readLocations * @param {string} localSocketAddr This node, never asked. * @param {object} ctx As peerComponentState, without `label`. @@ -277,7 +285,7 @@ async function componentStateOnOtherHolders(readLocations, localSocketAddr, ctx, try { locations = await readLocations(ctx.appName); } catch (error) { - log.warn(`${ctx.logPrefix} - ${ctx.appId}: the other holders cannot be read: ${error.message}`); + log.warn(`${ctx.logPrefix} - ${ctx.appName}: the other holders cannot be read: ${error.message}`); return PeerComponent.UNKNOWN; } const peers = (locations || []) diff --git a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js index 5736111607..18ee860517 100644 --- a/ZelBack/src/services/appMonitoring/peerFolderLiveness.js +++ b/ZelBack/src/services/appMonitoring/peerFolderLiveness.js @@ -76,7 +76,7 @@ async function holdingsRequest(socketAddr, challenge) { /** * A peer's answer as the callers read it. * @param {object} answer The `data` of the reply. - * @returns {{reachable: boolean, answerable: boolean, ready: boolean, folders: string[], holding: object, seeding: object}} + * @returns {{reachable: boolean, answerable: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[], heldOut: string[], volumeGone: string[]}} */ function readAnswer(answer) { // A peer that has not completed its first monitor pass cannot tell "I hold @@ -90,8 +90,16 @@ function readAnswer(answer) { // Where the peer stands in deciding a cold-start seed, per folder. Absent reads as // nowhere, for the same reasons. const seeding = (answer && typeof answer.seeding === 'object' && answer.seeding) || {}; + // The folders the peer holds and has not yet measured. Absent on a peer too old to + // say, which reads as none, as that node's answer always has. + const notLookedAt = Array.isArray(answer?.notLookedAt) ? answer.notLookedAt : []; + // The folders the peer cannot seed because their volume will not mount, with the + // volume's image on the node (heldOut) or gone (volumeGone). Absent on a peer too + // old to say, which reads as none, as that node's answer always has. + const heldOut = Array.isArray(answer?.heldOut) ? answer.heldOut : []; + const volumeGone = Array.isArray(answer?.volumeGone) ? answer.volumeGone : []; return { - reachable: true, answerable: true, ready, folders, holding, seeding, + reachable: true, answerable: true, ready, folders, holding, seeding, notLookedAt, heldOut, volumeGone, }; } @@ -101,7 +109,7 @@ function readAnswer(answer) { // may be the one holding the copy. function notReady() { return { - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } @@ -132,7 +140,7 @@ function notReady() { * somewhere else, and says nothing of the peer. * * @param {string} socketAddr Peer socket address - * @returns {Promise<{reachable: boolean, answerable: boolean, misrouted?: boolean, unproven?: boolean, ready: boolean, folders: string[], holding: object, seeding: object}>} + * @returns {Promise<{reachable: boolean, answerable: boolean, misrouted?: boolean, unproven?: boolean, ready: boolean, folders: string[], holding: object, seeding: object, notLookedAt: string[], heldOut: string[], volumeGone: string[]}>} */ async function probePeer(socketAddr) { const ip = extractIp(socketAddr); @@ -156,13 +164,13 @@ async function probePeer(socketAddr) { if (asked.verdict === IdentityVerdict.MISROUTED) { log.info(`peerFolderLiveness - a call to ${ip} was answered by ${asked.answeredAs}; nothing it holds is known`); return { - reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } if (asked.verdict === IdentityVerdict.UNREACHABLE) { log.info(`peerFolderLiveness - could not read ${ip}: ${asked.reason}`); return { - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } @@ -176,7 +184,7 @@ async function probePeer(socketAddr) { if (!asked.mayReadUnsigned) { log.info(`peerFolderLiveness - a reply from ${ip} does not prove it came from that node; nothing it holds is known`); return { - reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } @@ -196,12 +204,12 @@ async function probePeer(socketAddr) { } log.info(`peerFolderLiveness - ${ip} answered ${error.response.status} and cannot say which folders it holds`); return { - reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } log.info(`peerFolderLiveness - could not read ${ip}: ${error.message}`); return { - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }; } } diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js index 1aa2ff6506..0a845f9b0c 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderStateMachine.js @@ -5,7 +5,6 @@ const log = require('../../lib/log'); const dockerService = require('../dockerService'); const appReconciler = require('./appReconciler'); const appUninstaller = require('../appLifecycle/appUninstaller'); -const messageHelper = require('../messageHelper'); const syncthingService = require('../syncthingService'); const appQueryService = require('../appQuery/appQueryService'); const serviceHelper = require('../serviceHelper'); @@ -28,9 +27,11 @@ const { const { isPathMounted } = require('../utils/volumeService'); const globalState = require('../utils/globalState'); +const folderStates = require('./folderStates'); +const localChangeCheck = require('./localChangeCheck'); const { isSyncedRootName } = require('../appSystem/volumeReservedNames'); const primaryRole = require('../appLifecycle/primaryRole'); -const { componentStateOnOtherHolders } = require('./peerComponent'); +const { componentStateOnOtherHolders, PeerComponent } = require('./peerComponent'); const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); @@ -548,7 +549,8 @@ async function probeFolderSyncCompletion(folderId) { // so a folder of empty directories would read as holding data. const { globalFileBytes: globalBytes = 0, globalFiles = 0, inSyncFileBytes: inSyncBytes = 0, state, - receiveOnlyChangedFiles = 0, receiveOnlyChangedDirectories = 0, + receiveOnlyChangedFiles = 0, receiveOnlyChangedDirectories = 0, receiveOnlyChangedSymlinks = 0, + receiveOnlyChangedDeletes = 0, receiveOnlyChangedBytes = 0, } = await syncthingService.getDbStatusInFileBytes(folderId); const syncPercentage = globalBytes > 0 ? (inSyncBytes / globalBytes) * 100 : 100; @@ -567,6 +569,11 @@ async function probeFolderSyncCompletion(folderId) { // Directories are counted apart from files. A directory whose owner or mode // differs from the cluster's is a local change like any other. receiveOnlyChangedDirectories, + // The rest of what syncthing counts of this node's changes, which with the two + // above is what a check of them is stamped with: a pull moves none of them. + receiveOnlyChangedSymlinks, + receiveOnlyChangedDeletes, + receiveOnlyChangedBytes, // An EMPTY global index (globalBytes 0) means "unknown / not yet synced", // never "done": a node holding the only copy before its peers reconnect // reads globalBytes 0, and syncPercentage defaults to 100 there (vacuous). @@ -989,17 +996,23 @@ function seedClaimRanksFirst(a, b) { * @param {string} localSocketAddr This node's socket address * @param {Object} liveness This pass's peer view * @param {{address: string, bytes: *, newestModified: *}} own This node's claim + * @param {object} [options] + * @param {boolean} [options.writer] This node writes the app's other g: + * components. Only a peer whose folder already sends stops it then: a seed mark + * is an intent to seed, and a node that does not write the app cannot act on + * one - it never starts the component, so its folder never sends. * @returns {Promise<{ip: string, reason: string, outcome: string}|null>} */ -async function findSeedRival(appId, peers, localSocketAddr, liveness, own) { +async function findSeedRival(appId, peers, localSocketAddr, liveness, own, { writer = false } = {}) { const others = (peers || []).filter((peer) => peer?.ip && !socketAddressesMatch(peer.ip, localSocketAddr)); const answers = await Promise.all(others.map( async (peer) => ({ ip: peer.ip, ...await liveness.reread(peer.ip) }), )); const answered = answers.filter((answer) => answer.reachable && answer.answerable); - const decided = answered.find((answer) => answer.seeding?.[appId]?.stage === 'decided' + const decided = answered.find((answer) => (!writer && answer.seeding?.[appId]?.stage === 'decided') || (answer.ready && answer.folders.includes(appId))); if (decided) return { ip: decided.ip, reason: 'has already decided to seed it', outcome: 'yieldedToDecided' }; + if (writer) return null; const rival = answered.find((answer) => { const mark = answer.seeding?.[appId]; return (mark?.stage === 'intent' || mark?.stage === 'deciding') @@ -1291,33 +1304,141 @@ async function nudgeFolderDevices(folderId) { * version and deletes local-only items that are not ignored, so it runs only * against a complete global index (isSynced). * + * db/revert takes the whole folder, and deletes every change no other device holds. + * A folder is reverted only when both say its changes are not the app's data: + * - none of them is the only copy of data another device wrote, and + * - another holder runs the app, when any of them is a change this node made itself. + * A holder running the app is authoritative, and a change it does not have is + * this node's alone. No holder running it, this node holds the app's data, its + * earlier writes as primary included, and a revert has nothing to restore them + * from. Whether a holder runs it is asked on every pass that would revert, so a + * folder kept while none did is reverted once one does. + * A kept folder's changes do not hold the node back from taking over: promoted, it + * publishes them. Which changes the folder holds comes from localChangeCheck, and is + * acted on only while the folder is still the one it was checked against. Until then, + * and while whether another holder runs the app cannot be said, the folder is neither + * reverted nor promoted. The revert is localChangeCheck's, of the changes checked and + * no others. + * * @param {string} appId - The Syncthing folder ID * @param {Object|null} syncStatus - A probeFolderSyncCompletion status - * @returns {Promise} True when the folder had local changes and a revert was asked for + * @param {Object} folderCache - The folder's entry in receiveOnlySyncthingAppsCache: it + * carries the check's answer, and `keepingChanges` marks a folder already said to be + * keeping its changes, so that is said once, when it starts + * @param {Object} options + * @param {Function} options.isCurrent - Whether `folderCache` is still the folder's entry + * @param {Function|null} options.othersRunTheApp - What the app's other holders are doing + * with it, as a PeerComponent state; null when every holder runs it + * @returns {Promise} True when the folder has local changes it does not + * keep: a revert was asked for, or whether to revert cannot be said yet */ -async function revertLocalChangesIfSynced(appId, syncStatus) { +async function revertLocalChangesIfSynced(appId, syncStatus, folderCache, { isCurrent, othersRunTheApp }) { + const cache = folderCache; if (!syncStatus?.isSynced) return false; const changed = (syncStatus.receiveOnlyChangedFiles || 0) + (syncStatus.receiveOnlyChangedDirectories || 0); - if (changed === 0) return false; - log.warn(`revertLocalChangesIfSynced - ${appId} is synced but the receive-only folder has ${changed} locally changed item(s) (${syncStatus.receiveOnlyChangedFiles || 0} files, ${syncStatus.receiveOnlyChangedDirectories || 0} directories); reverting to the cluster's version`); - try { - // dataOrThrow: dbRevert answers in-band; without it this catch is - // dead code and a failed revert reads as reverted - messageHelper.dataOrThrow(await syncthingService.dbRevert(appId)); - fluxEventBus.publish('syncthing:localChangesReverted', { - folder: appId, - files: syncStatus.receiveOnlyChangedFiles || 0, - directories: syncStatus.receiveOnlyChangedDirectories || 0, + if (changed === 0) { + delete cache.keepingChanges; + delete cache.localChangeCheck; + return false; + } + const files = syncStatus.receiveOnlyChangedFiles || 0; + const directories = syncStatus.receiveOnlyChangedDirectories || 0; + const devices = (await syncthingService.getConfigFolders(appId).catch(() => null))?.devices; + if (!Array.isArray(devices)) { + log.warn(`revertLocalChangesIfSynced - ${appId}: could not read which devices share the folder; neither reverting nor promoting this pass`); + return true; + } + const stamp = localChangeCheck.stampOf(syncStatus, devices); + const answer = localChangeCheck.answerFor(cache, stamp); + if (!answer) { + localChangeCheck.request(appId, cache, stamp, isCurrent); + return true; + } + const keep = (reason, why) => { + if (cache.keepingChanges === reason) return false; + cache.keepingChanges = reason; + log.warn(`revertLocalChangesIfSynced - ${appId} ${why} (${files} files, ${directories} directories changed); keeping its changes, a revert would delete them`); + fluxEventBus.publish('syncthing:localChangesKept', { + folder: appId, files, directories, reason, }); - } catch (error) { - log.error(`revertLocalChangesIfSynced - revert of local changes for ${appId} failed: ${error.message}`); + return false; + }; + if (answer.onlyCopy) { + delete cache.waitingOnHolders; + return keep('onlyCopy', `holds a local change to data from another device that no other device holds (${answer.onlyCopy})`); } + if (answer.madeHere && othersRunTheApp) { + const elsewhere = await othersRunTheApp(); + if (elsewhere === PeerComponent.NOT_RUNNING) { + delete cache.waitingOnHolders; + return keep('noHolderRunsTheApp', 'holds changes it made itself while no other holder runs the app'); + } + if (elsewhere !== PeerComponent.RUNNING) { + if (!cache.waitingOnHolders) log.info(`revertLocalChangesIfSynced - ${appId}: whether another holder runs the app cannot be said; neither reverting nor promoting until it can`); + cache.waitingOnHolders = true; + return true; + } + } + delete cache.waitingOnHolders; + delete cache.keepingChanges; + localChangeCheck.revert(appId, cache, answer, isCurrent, { files, directories }); return true; } +/** + * What a candidate holds of an app's g: components, for ranking the seed of one + * of their folders: the bytes of all of them, and the newest write to any. + * + * An app whose g: components are seeded on different nodes cannot start: its + * primary runs them all, and no node holds every copy. Ranked on the whole app, + * every folder of it names the same node. A candidate missing a figure for any + * of the folders has not answered. + * @param {object|null|undefined} folderClaim What it holds of this folder. + * @param {Array} siblingClaims What it holds of each other g: folder. + * @returns {object|null|undefined} The claim to rank; this folder's alone when the app has one g: component. + */ +function electedAppClaim(folderClaim, siblingClaims) { + if (!siblingClaims.length) return folderClaim; + const parts = [folderClaim, ...siblingClaims]; + if (!parts.every(isRankableClaim)) return null; + return { + bytes: parts.reduce((total, part) => total + part.bytes, 0), + newestModified: Math.max(...parts.map((part) => part.newestModified)), + }; +} + +/** + * Which node writes an app's other g: components: this one, a peer whose folder + * for one of them sends, or none. + * + * The app's primary runs all of its g: components, so a folder of one of them + * that nothing has seeded - a component the spec has just added - is the + * writer's to seed, wherever the address order falls. Seeded anywhere else, the + * writer never holds a copy it may run it on, and the node that does is not the + * primary. + * @param {string[]} siblings Folder ids of the app's other g: components. + * @param {object[]} electionList The holders in the election. + * @param {string} localSocketAddr + * @param {object} liveness Peer folder liveness. + * @returns {Promise<{here: boolean, ip?: string}|null>} + */ +async function electedAppWriter(siblings, electionList, localSocketAddr, liveness) { + const heldHere = await Promise.all(siblings.map((sibling) => appQueryService.holdsComponent(sibling))); + if (heldHere.includes(true)) return { here: true }; + const others = electionList.filter((peer) => peer?.ip && !socketAddressesMatch(peer.ip, localSocketAddr)); + const answers = await Promise.all(others.map( + async (peer) => ({ ip: peer.ip, ...await liveness.read(peer.ip) }), + )); + const writer = answers.find((answer) => answer.reachable && answer.ready + && siblings.some((sibling) => (answer.folders || []).includes(sibling))); + return writer ? { here: false, ip: writer.ip } : null; +} + /** * Handle receive-only to send-receive transition * @param {Object} params - Parameters + * @param {string[]} [params.electedSiblings] Folder ids of the app's other g: + * components, for a g: folder; empty for any other. * @returns {Promise} Updated folder config and cache */ async function handleReceiveOnlyTransition(params) { @@ -1330,9 +1451,13 @@ async function handleReceiveOnlyTransition(params) { unsyncedSubdirs, syncthingFolder, liveness, + electedSiblings = [], + receiveOnlySyncthingAppsCache, + othersRunTheApp = null, } = params; const folderPath = syncthingFolder.path || `${appsFolder}${appId}/appdata`; + await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_HOLDINGS, appId); // Whether any CONNECTED peer genuinely holds the data. Gates the election (a true // cold start - nobody serving - must still elect one seed instead of standing off) @@ -1360,15 +1485,11 @@ async function handleReceiveOnlyTransition(params) { // not a smaller claim, it is no claim - and standDownOnUnknown below refuses to lead // on exactly that. Left up, the last good figure says 5.8 GB to every peer while this // node refuses to act on it: they rank it first and defer, it defers to its own - // unknown, and nobody seeds for as long as the read keeps failing. Removed, a peer - // reads it as not having answered, which drops the field to the address order - the - // case that already exists for a peer too old to answer at all. - if (holdings) { - if (!globalState.folderHoldings) globalState.folderHoldings = new Map(); - globalState.folderHoldings.set(appId, holdings); - } else { - globalState.folderHoldings?.delete(appId); - } + // unknown, and nobody seeds for as long as the read keeps failing. Recorded as no + // claim, it drops the field to the address order - the case that already exists for + // a peer too old to answer at all - and the folder is not one this node has yet to + // look at. + folderStates.measured(appId, holdings); // Designated-leader election, debounced: require leadership to hold for // LEADER_CONFIRM_COUNT consecutive cycles, so a single transient peer-visibility blip // doesn't flip a follower to leader. @@ -1394,8 +1515,10 @@ async function handleReceiveOnlyTransition(params) { // expires - 125 minutes with the app down. Dropped from the list here, before the // pick, when this node can show the holder is gone rather than merely silent to it. const electionList = await holderListExcludingDead(appId, runningAppList, localSocketAddr, liveness); - // What each candidate says it holds for THIS folder, from the probe the pass has - // already paid for. This node answers for itself rather than asking itself. + // What each candidate says it holds for THIS folder - and for an app with more + // than one g: component, for all of their folders (electedAppClaim) - from the + // probe the pass has already paid for. This node answers for itself rather than + // asking itself. // // Every claim is filed under the candidate's own address as the election list // carries it, because that is the string bestHolder looks it up by, and an object @@ -1403,13 +1526,38 @@ async function handleReceiveOnlyTransition(params) { // decides where the answer comes from and nothing else - keyed from anywhere but // the list, a node stops finding its own claim the moment the two spellings of // its address differ, and the ranking degrades to the address order silently. + const ownClaim = electedAppClaim(holdings, electedSiblings.map((sibling) => folderStates.claimOf(sibling))); const claims = {}; + // Candidates that hold this app and have not yet measured what they hold of it - an + // install their monitor has not reached. Ranked now, one would read as holding + // nothing and the field would fall to the address order over a holder that may hold + // the owner's data, so nobody seeds this pass. + const notLookedAt = []; + // Candidates that cannot seed this app because a volume of theirs will not mount, + // and so are left out of the choice. One whose volume image is still on it may hold + // the owner's data, so nobody seeds over it unless another candidate holds data; one + // whose image is gone holds nothing. + const volumeHeldOut = []; + const volumeGone = []; await Promise.all(electionList.map(async (peer) => { if (socketAddressesMatch(peer.ip, localSocketAddr)) { - if (holdings) claims[peer.ip] = holdings; + if (ownClaim) claims[peer.ip] = ownClaim; return; } const answer = await liveness.read(peer.ip); + const listed = (field) => answer?.ready && [appId, ...electedSiblings].some((folder) => (answer[field] ?? []).includes(folder)); + if (listed('notLookedAt')) { + notLookedAt.push(peer.ip); + return; + } + if (listed('heldOut')) { + volumeHeldOut.push(peer.ip); + return; + } + if (listed('volumeGone')) { + volumeGone.push(peer.ip); + return; + } // Only a READY peer's claim counts - before its first monitor pass a node cannot // tell "I hold nothing" from "I have not looked". Defence in depth rather than // the thing that enforces it: a peer that has not determined its folder state @@ -1417,7 +1565,8 @@ async function handleReceiveOnlyTransition(params) { // an outcome. Deliberately untested for that reason - a test would pass whether // this condition were here or not. if (answer?.ready && answer.holding && answer.holding[appId]) { - claims[peer.ip] = answer.holding[appId]; + const claim = electedAppClaim(answer.holding[appId], electedSiblings.map((sibling) => answer.holding[sibling])); + if (claim) claims[peer.ip] = claim; } })); // A node that cannot read its OWN holdings must not outrank a peer that can show it @@ -1428,8 +1577,31 @@ async function handleReceiveOnlyTransition(params) { const someoneElseHolds = Object.entries(claims) .some(([ip, claim]) => !socketAddressesMatch(ip, localSocketAddr) && (claim?.bytes || 0) > 0); const standDownOnUnknown = ownHoldingsUnknown && someoneElseHolds; - const electedLeader = !standDownOnUnknown - && isDesignatedLeader(electionList, localSocketAddr, aPeerHasData, claims); + // The writer of the app's other g: components seeds this folder when no peer + // serves it, and no other node does. What a standby holds in it and never sent + // is a local change against the app's primary, reverted as any standby's is. + const appWriter = electedSiblings.length + ? await electedAppWriter(electedSiblings, electionList, localSocketAddr, liveness) + : null; + if (notLookedAt.length) { + fluxEventBus.count('syncthing:seedDecision', appId, 'holderNotLookedAt'); + if (!cache.waitingOnLook) log.info(`handleReceiveOnlyTransition - ${appId}: ${notLookedAt.join(', ')} has not yet measured what it holds; nobody seeds until it has`); + } + cache.waitingOnLook = notLookedAt.length > 0; + const leftOut = [...volumeHeldOut, ...volumeGone]; + const candidates = electionList.filter((peer) => !leftOut.some((ip) => socketAddressesMatch(ip, peer.ip))); + const candidateHolds = candidates.some((peer) => (claims[peer.ip]?.bytes || 0) > 0); + const waitingOnVolume = volumeHeldOut.length > 0 && !candidateHolds; + if (leftOut.length) fluxEventBus.count('syncthing:seedDecision', appId, 'holderCannotSeed'); + if (waitingOnVolume) { + fluxEventBus.count('syncthing:seedDecision', appId, 'holderVolumeHeldOut'); + if (!cache.waitingOnVolume) log.info(`handleReceiveOnlyTransition - ${appId}: ${volumeHeldOut.join(', ')} cannot mount a volume that may hold the app's data and no other holder has data; nobody seeds until it mounts`); + } + cache.waitingOnVolume = waitingOnVolume; + const electedLeader = appWriter + ? appWriter.here && !aPeerHasData + : !standDownOnUnknown && !notLookedAt.length && !waitingOnVolume + && isDesignatedLeader(candidates, localSocketAddr, aPeerHasData, claims); // The floor holderIsGone asks of a silent holder, asked of this node before // its own win can count: a node whose peers have gone quiet is the one that // fell over, and a win it confirms in that state seeds the app on a @@ -1441,12 +1613,13 @@ async function handleReceiveOnlyTransition(params) { const isLeader = electedLeader && cache.leaderStreak >= LEADER_CONFIRM_COUNT; // Announced on the pass this node is elected, so a rival that confirms on the same // pass as it can see it - see findSeedRival. + const seedHoldings = electedSiblings.length ? ownClaim : holdings; const ownSeedClaim = { address: (runningAppList || []).find((peer) => socketAddressesMatch(peer?.ip, localSocketAddr))?.ip ?? localSocketAddr, - bytes: holdings?.bytes ?? null, - newestModified: holdings?.newestModified ?? null, + bytes: seedHoldings?.bytes ?? null, + newestModified: seedHoldings?.newestModified ?? null, }; - if (electedLeader && connected) markSeed(appId, 'intent', holdings); + if (electedLeader && connected) markSeed(appId, 'intent', seedHoldings); else globalState.seedMarks.delete(appId); // Withdrawn on every unpromoted pass, so a lost election drops the claim // and the intent behind it. It is raised again only where the promotion is @@ -1492,15 +1665,15 @@ async function handleReceiveOnlyTransition(params) { if (isLeader && !blocker?.holdsWritableCopy) { // Marked before the peers are read, so a rival deciding at the same moment // sees this node. Every way out below that does not seed goes back to intent. - markSeed(appId, 'deciding', holdings); + markSeed(appId, 'deciding', seedHoldings); const notSeeding = () => { - markSeed(appId, 'intent', holdings); + markSeed(appId, 'intent', seedHoldings); decideType(syncthingFolder, containerDataFlags, 'receiveonly'); return { syncthingFolder, cache }; }; await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_SEED_DECISION, appId); const decisionStartedAt = monotonicMs(); - const rival = await findSeedRival(appId, runningAppList, localSocketAddr, liveness, ownSeedClaim); + const rival = await findSeedRival(appId, runningAppList, localSocketAddr, liveness, ownSeedClaim, { writer: appWriter?.here === true }); if (rival) { fluxEventBus.count('syncthing:seedDecision', appId, rival.outcome); log.info(`handleReceiveOnlyTransition - ${appId} is elected to seed but ${rival.ip} ${rival.reason}; going on as a standby`); @@ -1560,7 +1733,7 @@ async function handleReceiveOnlyTransition(params) { log.info(`handleReceiveOnlyTransition - ${appId}: deciding to seed took ${decisionMs}ms, longer than a pass (${SEED_DECISION_BUDGET_MS}ms); deciding again on the next pass`); return notSeeding(); } - markSeed(appId, 'decided', holdings); + markSeed(appId, 'decided', seedHoldings); fluxEventBus.count('syncthing:seedDecision', appId, 'seeded'); // Every gate passed - but deciding the promotion is not applying it. The @@ -1605,7 +1778,8 @@ async function handleReceiveOnlyTransition(params) { const reason = `candidates=${electionList.length} self=${selfInElection} ` + `elected=${electedLeader} connected=${connected} ` + `streak=${cache.leaderStreak}/${LEADER_CONFIRM_COUNT} ` - + `peerHasData=${aPeerHasData}`; + + `peerHasData=${aPeerHasData}` + + (electedSiblings.length ? ` appWriter=${appWriter ? (appWriter.here ? 'here' : appWriter.ip) : 'none'}` : ''); if (reason !== cache.lastNotPromotedReason) { log.info(`handleReceiveOnlyTransition - ${appId} not promoted: ${reason}`); cache.lastNotPromotedReason = reason; @@ -1637,9 +1811,12 @@ async function handleReceiveOnlyTransition(params) { // data: local additions in a receiveonly folder leave needBytes 0 / completion 100, // and promoting would broadcast those local changes cluster-wide. Verify the folder // is clean first; if not, revert the local changes (db/revert undoes local edits in - // a receiveonly folder) and promote on a later cycle once verifiably clean. The - // leader path above is exempt by design - the leader's local data IS the seed. - if (await revertLocalChangesIfSynced(appId, syncStatus)) { + // a receiveonly folder) and promote on a later cycle once verifiably clean. Changes + // no other device holds are kept, and promoted with the rest of this node's data. + // The leader path above is exempt by design - the leader's local data IS the seed. + if (await revertLocalChangesIfSynced(appId, syncStatus, cache, { + isCurrent: () => receiveOnlySyncthingAppsCache.get(appId) === cache, othersRunTheApp, + })) { return { syncthingFolder, cache }; } if (syncStatus.isSynced) { @@ -1824,12 +2001,22 @@ async function manageFolderSyncState(params) { syncthingFolder, installedAppName, liveness, + electedSiblings = [], } = params; // One per pass per folder, whatever the pass decides: a harness suite counts // passes from an event to know every holder has decided since it. fluxEventBus.count('syncthing:folderPass', appId, 'evaluated'); + // What the app's other holders are doing with its g: components. Every one of + // them is asked about: a holder running any of them is its primary. + const othersHold = () => componentStateOnOtherHolders(readAppLocation, localSocketAddr, { + appIds: [appId, ...electedSiblings], identifier, appName: installedAppName, liveness, logPrefix: 'manageFolderSyncState', + }); + // Whether a change only this node made, and no other device holds, is reverted + // turns on whether another holder runs the app. Every holder of an r: folder runs it. + const othersRunTheApp = containerDataFlags.includes('g') ? othersHold : null; + // Check if folder already exists and is in sendreceive mode const folderAlreadySyncing = syncFolder && syncFolder.type === 'sendreceive'; const sendingSingleWriter = folderAlreadySyncing && containerDataFlags.includes('g'); @@ -1846,9 +2033,6 @@ async function manageFolderSyncState(params) { // its process, and what it holds must not go out until the election says // it is primary again. log.info(`manageFolderSyncState - ${appId} is sendreceive and not held here, demoting until the election decides`); - const othersHold = () => componentStateOnOtherHolders(readAppLocation, localSocketAddr, { - appId, identifier, appName: installedAppName, liveness, logPrefix: 'manageFolderSyncState', - }); await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_BEFORE_STANDBY_HOLD, appId); await primaryRole.holdAsStandby(identifier, appId, { othersHold }); return { syncthingFolder, cache: { restarted: false, numberOfExecutions: 0 } }; @@ -1905,6 +2089,9 @@ async function manageFolderSyncState(params) { unsyncedSubdirs, syncthingFolder, liveness, + electedSiblings, + receiveOnlySyncthingAppsCache, + othersRunTheApp, }); return result; } @@ -1943,6 +2130,7 @@ async function manageFolderSyncState(params) { } // Default case - ensure container is running + folderStates.settled(appId); await ensureContainerRunning(appId, containerDataFlags); if (containerDataFlags.includes('g')) { // The ready mark the election reads has to outlive this pass. @@ -1950,7 +2138,9 @@ async function manageFolderSyncState(params) { // deviation (an owner or mode included) is never overwritten by syncthing // in a receiveonly folder, and the primary counts it as outstanding. if (syncFolder?.type === 'receiveonly') { - await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId)); + await revertLocalChangesIfSynced(appId, await getFolderSyncCompletion(appId), cache, { + isCurrent: () => receiveOnlySyncthingAppsCache.get(appId) === cache, othersRunTheApp, + }); } return { syncthingFolder, cache }; } diff --git a/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js b/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js index 453bb9f3db..89edeed48a 100644 --- a/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js +++ b/ZelBack/src/services/appMonitoring/syncthingFolderWrites.js @@ -2,6 +2,7 @@ const log = require('../../lib/log'); const serviceHelper = require('../serviceHelper'); const syncthingService = require('../syncthingService'); const globalState = require('../utils/globalState'); +const folderStates = require('./folderStates'); const fluxEventBus = require('../utils/fluxEventBus'); const { appsFolder } = require('../utils/appConstants'); const { OWNED_FOLDER_SETTINGS } = require('./syncthingMonitorHelpers'); @@ -81,9 +82,8 @@ function recordType(folderId, type) { writable?.add(folderId); // What a folder holds that the cluster's index does not is a receive-only // question, and promotion answers it: everything this node holds is now - // published. Dropped rather than zeroed - absent is what a peer reads as - // "not a receive-only holder". - globalState.folderHoldings?.delete(folderId); + // published. + folderStates.sending(folderId); // A folder that sends has been seeded; peers read that from the folder itself. globalState.seedMarks.delete(folderId); } else { @@ -491,6 +491,7 @@ async function changeSyncthingFolderType(folderId, folderType, { module.exports = { FOLDER_TYPE_SETTLE_MS, changeSyncthingFolderType, + exclusive, folderConfig, pauseAllFolders, putFolders, diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitor.js b/ZelBack/src/services/appMonitoring/syncthingMonitor.js index 3bedcac507..2ebc550c7e 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitor.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitor.js @@ -38,6 +38,7 @@ const volumeService = require('../utils/volumeService'); const appTamperingDetectionService = require('../appTamperingDetectionService'); const appReconciler = require('./appReconciler'); const syncthingFolderWrites = require('./syncthingFolderWrites'); +const folderStates = require('./folderStates'); const primaryRole = require('../appLifecycle/primaryRole'); const { manageFolderSyncState, @@ -52,6 +53,12 @@ const syncthingEventsConsumer = require('./syncthingEventsConsumer'); // Global collections const globalAppsLocations = config.database.appsglobal.collections.appsLocations; +// How long a device no folder uses keeps its entry after syncthing last saw it. +// Its connection record is what proves a peer gone (peerFolderLiveness), and a +// standby that remembers a primary no location names any more still has to prove +// that primary gone before it starts the app; with the entry deleted it never can. +const UNUSED_DEVICE_KEEP_MS = 24 * 60 * 60 * 1000; + // Path constants const fluxDirPath = process.env.FLUXOS_PATH || path.join(process.env.HOME, 'zelflux'); const appsFolderPath = process.env.FLUX_APPS_FOLDER || path.join(fluxDirPath, 'ZelApps'); @@ -74,13 +81,16 @@ const appsFolder = `${appsFolderPath}/`; * @param {string} appFolder - App folder path * @param {boolean} sending - Whether syncthing currently holds this folder sendreceive * @param {string[]} unsyncedSubdirs - volume-root names the spec declared with ml: - * @returns {Promise<{isSafe: boolean, reason: string}>} Result after any repair + * @returns {Promise<{isSafe: boolean, reason: string, volumeFileMissing?: boolean}>} Result + * after any repair; volumeFileMissing when the mount found conclusively that the + * volume's image is not on this node */ async function verifyAppFolderMountWithRepair(appId, appFolder, sending, unsyncedSubdirs = []) { const verify = sending ? verifySendReceiveFolderSafety : verifyFolderMountSafety; let mountSafety = await verify(appId, appFolder, unsyncedSubdirs); if (!mountSafety.isSafe && !mountSafety.isMounted) { const mountAttempt = await volumeService.ensureAppVolumeMounted(appId); + if (mountAttempt.reason === 'volume_file_missing') return { ...mountSafety, volumeFileMissing: true }; if (mountAttempt.mounted) { // This pass replaces the record when it mounts from somewhere other than // where the record puts it, so a later one cannot re-derive the fact - @@ -128,8 +138,8 @@ function appComponents(installedApp) { * @param {Array<{appId: string, appName: string}>} extraFolders - Folder entries * verified by id alone, for folders whose owning app's spec cannot be read * @returns {Promise<{unmountedApps: Array, verifiedSafeIds: string[]}>} Apps with - * unmounted folders, and the folder ids that verified safe (so a pending - * mount-verify flag on them can be resolved) + * unmounted folders, each saying whether its volume image is gone, and the folder + * ids that verified safe (so a pending mount-verify flag on them can be resolved) */ async function checkAppFolderMounts(appsInstalled, sendingFolderIds, extraFolders = []) { const unmountedApps = []; @@ -142,7 +152,9 @@ async function checkAppFolderMounts(appsInstalled, sendingFolderIds, extraFolder verifiedSafeIds.push(appId); } else { // Folder exists but mount is not safe (empty and not mounted - likely unmounted loop device) - unmountedApps.push({ appId, appName, reason: mountSafety.reason }); + unmountedApps.push({ + appId, appName, reason: mountSafety.reason, volumeFileMissing: !!mountSafety.volumeFileMissing, + }); } }; @@ -268,6 +280,8 @@ async function appLocation(appName) { * This function handles both legacy apps (version <= 3) and newer apps (version > 3) * * @param {Object} params - Parameters object + * @param {string[]} [params.electedFolderIds] Folder ids of the app's g: + * components, which its primary runs together. * @returns {Promise} */ async function processContainerData(params) { @@ -286,6 +300,8 @@ async function processContainerData(params) { foldersConfiguration, newFoldersConfiguration, liveness, + electedFolderIds = [], + unsafeFolderIds, } = params; const containersData = containerData.split('|'); @@ -312,6 +328,7 @@ async function processContainerData(params) { if (!markerReady) { log.warn(`processContainerData - ${appId} volume not mounted; skipping syncthing configuration this cycle`); fluxEventBus.count('syncthing:folderPass', appId, 'volumeNotMounted'); + folderStates.notMounted(appId); return; } @@ -329,7 +346,19 @@ async function processContainerData(params) { } // Get and process app locations - let locations = await appLocation(installedAppName); + let locations; + try { + locations = await readAppLocation(installedAppName); + } catch (error) { + // Without its locations the folder would be shared with no peer, and the device + // sweep would remove every peer it has: it is held out of this pass instead. What + // this node tells its peers of the folder stays as it was: a read that failed says + // nothing about what the volume holds. + log.warn(`processContainerData - ${appId} locations could not be read (${error.message}); leaving the folder as it is this cycle`); + fluxEventBus.count('syncthing:folderPass', appId, 'locationsUnread'); + unsafeFolderIds.add(appId); + return; + } locations = sortAndFilterLocations(locations, localSocketAddr); // Build device configuration (parallelized internally) @@ -365,6 +394,7 @@ async function processContainerData(params) { syncthingFolder, installedAppName, liveness, + electedSiblings: electedFolderIds.includes(appId) ? electedFolderIds.filter((folderId) => folderId !== appId) : [], }); // Update cache if provided @@ -518,6 +548,11 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // missing" by it, and the sweep at the end deletes by it. const ownerIds = syncingFolderOwnerIds(appsInstalled.data); + // Every receive-only folder of an installed app has a state before this node first + // answers its peers below. After a restart the states start empty while the volumes + // still hold their data, so each folder is not looked at until this pass measures it. + appsInstalled.data.forEach((installedApp) => folderStates.installed(installedApp)); + // Get required IDs and configurations const localDeviceId = await syncthingService.getDeviceId(); if (!localDeviceId) { @@ -569,6 +604,13 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { await fluxEventBus.checkpoint(fluxEventBus.Checkpoint.SYNCTHING_AFTER_FOLDER_READ); + // A receive-only folder of an app whose spec cannot be read is one this pass + // does not measure, so what it holds is unknown. Its folders come from + // syncthing's own list, as the sweep below takes them. + allFolders + .filter((folder) => folder.type === 'receiveonly' && ownedByUnreadableApp(folder.id)) + .forEach((folder) => folderStates.specUnreadable(folder.id)); + // Publish which folders this node holds writable, for the peers that ask before // promoting one of their own. Recorded here rather than read on demand: the // answer is a byproduct of a pass the monitor already makes, so serving it costs @@ -711,8 +753,11 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // re-promotes once the mount is healthy. // eslint-disable-next-line no-restricted-syntax const unreadableIds = new Set(unreadableFolderEntries.map(({ appId }) => appId)); - for (const { appId, reason } of unmountedApps) { + for (const { appId, reason, volumeFileMissing } of unmountedApps) { unsafeFolderIds.add(appId); + if (volumeFileMissing) folderStates.volumeGone(appId); + else folderStates.heldOut(appId); + fluxEventBus.count('syncthing:folderPass', appId, 'mountUnsafe'); // The container is the other writer over the bad mount, and holding it // asks nothing of syncthing: a component that syncs the folder, or one // whose spec cannot be read to say it does not, is held before the @@ -802,6 +847,7 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { folderIds, foldersConfiguration, newFoldersConfiguration, + unsafeFolderIds, }; // Process all installed apps @@ -834,6 +880,9 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { }); } else { // Newer app (version > 3) - compose with multiple components + // The folders of the app's g: components, which its primary runs together. + const electedFolderIds = mountParser.gComponentIdentifiers(installedApp) + .map((electedIdentifier) => dockerService.getAppIdentifier(electedIdentifier)); // eslint-disable-next-line no-restricted-syntax for (const installedComponent of installedApp.compose) { const identifier = `${installedComponent.name}_${installedApp.name}`; @@ -844,6 +893,7 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { containerData: installedComponent.containerData, identifier, installedAppName: installedApp.name, + electedFolderIds, }); } } @@ -876,8 +926,23 @@ async function syncthingAppsCore(state, installedAppsFn, getGlobalStateFn) { // A peer device cannot be attributed to an app without doing that app's // work, so while anything is held out the device sweep stands down entirely // rather than delete a peer it simply never saw. - const nonUsedDevices = unsafeFolderIds.size > 0 ? [] : allDevices.filter( - (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId, + // A device seen within UNUSED_DEVICE_KEEP_MS stays too, unless another device + // is configured under its name - then it is not the device at that address - + // and every device stays while when each was last seen cannot be read. + // A device any folder lists stays as well: syncthing drops a deleted device + // from every folder that lists it, outside the folder's writes here, and a + // revert of a standby's changes holds the folder's devices fixed only against + // those writes. It leaves its folders through the folder writes below, and is + // deleted by the first pass that finds it in none. + const deviceStats = unsafeFolderIds.size > 0 ? null : await syncthingService.getDeviceStats().catch(() => null); + const seenRecently = (deviceID) => Date.now() - Date.parse(deviceStats?.[deviceID]?.lastSeen) < UNUSED_DEVICE_KEEP_MS; + const nameHeldByAnother = (device) => Boolean(device.name) + && allDevices.some((other) => other.deviceID !== device.deviceID && other.name === device.name); + const listedInAFolder = new Set(allFolders.flatMap((folder) => (folder.devices ?? []).map(({ deviceID }) => deviceID))); + const nonUsedDevices = !deviceStats ? [] : allDevices.filter( + (syncthingDevice) => !devicesIds.includes(syncthingDevice.deviceID) && syncthingDevice.deviceID !== localDeviceId + && !listedInAFolder.has(syncthingDevice.deviceID) + && (!seenRecently(syncthingDevice.deviceID) || nameHeldByAnother(syncthingDevice)), ); // Parallelize cleanup operations diff --git a/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js b/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js index b5697cb837..0f726d5c34 100644 --- a/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js +++ b/ZelBack/src/services/appMonitoring/syncthingMonitorHelpers.js @@ -56,15 +56,21 @@ async function getDeviceID(fluxIP, retries = 0) { * folder syncing with a device that holds no copy of it. A peer that cannot * prove either way is asked the unsigned way. * + * A node listed at `name` that gives no device - its FluxOS does not answer, or + * its syncthing has not answered it - keeps the device syncthing has configured + * under `name`, so a holder stays in its folders for as long as it is listed. + * * Every call counts where its answer came from under `syncthing:deviceSource`, - * keyed by `name`: verified, verifiedNoDevice, introduced, withheld, held or - * unsigned. + * keyed by `name`: verified, verifiedNoDevice, introduced, withheld, held, + * unsigned or configured. * * @param {string} name - Device name (IP:port) * @param {Map} cache - Cache map + * @param {string|null} [configured] - The device syncthing has configured under + * `name`, when exactly one is * @returns {Promise} Device ID or null */ -async function getDeviceIDCached(name, cache) { +async function getDeviceIDCached(name, cache, configured = null) { const { IdentityVerdict } = peerIdentityService; const from = (source, deviceID) => { fluxEventBus.count('syncthing:deviceSource', name, source); @@ -76,7 +82,10 @@ async function getDeviceIDCached(name, cache) { const signedDeviceID = result.identity.deviceId; // Verified with no device: that node's syncthing has not answered it yet. // What was held stands until it has. - if (!signedDeviceID) return from('verifiedNoDevice', cache.get(name) ?? null); + if (!signedDeviceID) { + if (cache.has(name)) return from('verifiedNoDevice', cache.get(name)); + return configured ? from('configured', configured) : from('verifiedNoDevice', null); + } cache.set(name, signedDeviceID); return from('verified', signedDeviceID); } @@ -100,8 +109,9 @@ async function getDeviceIDCached(name, cache) { const deviceID = await getDeviceID(name); if (deviceID) { cache.set(name, deviceID); + return from('unsigned', deviceID); } - return from('unsigned', deviceID); + return configured ? from('configured', configured) : from('unsigned', null); } /** @@ -184,7 +194,9 @@ async function buildDeviceConfiguration( const addresses = peerSyncthingAddresses(ip, port); const name = `${ip}:${port}`; - const deviceID = await getDeviceIDCached(name, deviceCache); + const underName = allDevices.filter((device) => device.name === name); + const configured = underName.length === 1 ? underName[0].deviceID : null; + const deviceID = await getDeviceIDCached(name, deviceCache, configured); if (!deviceID) { return null; diff --git a/ZelBack/src/services/appNetwork/portManager.js b/ZelBack/src/services/appNetwork/portManager.js index 479ac7c33a..7f02a99c5f 100644 --- a/ZelBack/src/services/appNetwork/portManager.js +++ b/ZelBack/src/services/appNetwork/portManager.js @@ -11,12 +11,14 @@ const upnpService = require('../upnpService'); const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const fluxHttpTestServer = require('../utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../utils/fluxUdpTestServer'); const { localAppsInformation, globalAppsInformation } = require('../utils/appConstants'); const appUtilities = require('../utils/appUtilities'); const { Privilege, authOf } = require('../utils/privileges'); const fluxCaching = require('../utils/cacheManager'); const globalState = require('../utils/globalState'); const fluxEventBus = require('../utils/fluxEventBus'); +const policyStore = require('../policyStore'); const { nodeSigner } = require('../utils/nodeSigner'); // Global cache for failed nodes @@ -298,7 +300,7 @@ async function restoreAppsPortsSupport() { // eslint-disable-next-line no-restricted-syntax for (const port of application.ports) { // eslint-disable-next-line no-await-in-loop - let upnpOk = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`); + let upnpOk = (await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`)).mapped; if (!upnpOk) { if (!retryDelayElapsed) { // eslint-disable-next-line no-await-in-loop @@ -306,7 +308,7 @@ async function restoreAppsPortsSupport() { retryDelayElapsed = true; } // eslint-disable-next-line no-await-in-loop - upnpOk = await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`); + upnpOk = (await upnpService.mapUpnpPort(serviceHelper.ensureNumber(port), `Flux_App_${application.name}`)).mapped; } if (!upnpOk) { failedPort = port; @@ -734,6 +736,22 @@ async function askSiblingsForHeldPort(appPorts, localSocketAddress) { return null; } +/** + * The signed policy document that turns the pre-install refusal over UDP off. + * Published as `true`, a port two peers heard nothing on over UDP installs as + * `udpUnproven`, its UDP still reported in /flux/health; absent or anything + * else, the refusal stands. + */ +const UDP_INSTALL_REFUSAL_OFF_DOCUMENT = 'udpinstallrefusaloff'; + +/** + * Whether the signed policy has turned the pre-install refusal over UDP off. + * @returns {boolean} + */ +function udpInstallRefusalOff() { + return policyStore.getDocument(UDP_INSTALL_REFUSAL_OFF_DOCUMENT) === true; +} + /** * How many independent peers must agree before a port test refuses an install. * Not config: it is a property of what counts as evidence, not a tuning knob, @@ -808,6 +826,81 @@ function portNotOurs(portsToTest, answered, token) { return at === -1 ? null : portsToTest[at]; } +/** + * Reads one peer's UDP answers: adds the ports our token came back from to + * those proven so far, and names the ports this peer heard nothing of ours on. + * + * Our token is proof, whichever peer heard it. Silence, or an answer without + * our token, is this peer's evidence against a port, and it stands only while + * no peer has proven that port: UDP loses datagrams, so one peer's silence on a + * port another peer heard our token from is loss. A port missing from the + * answers is one the peer could not send to, which says nothing. + * + * @param {number[]} portsToTest - the ports asked about + * @param {object} udpAnswered - port -> what that port replied, or null, from the peer + * @param {string} token - the secret this node's test servers answer with + * @param {Set} proven - ports proven so far; added to + * @returns {number[]} the ports this peer heard nothing of ours on + */ +function udpPortsNotOurs(portsToTest, udpAnswered, token, proven) { + const closed = []; + portsToTest.forEach((port) => { + const probed = port >= config.fluxapps.portMin && port <= config.fluxapps.portMax; + if (!probed) return; + + const reply = udpAnswered[port] !== undefined ? udpAnswered[port] : udpAnswered[String(port)]; + if (reply === undefined) return; + + if (typeof reply === 'string' && reply.includes(token)) { + proven.add(port); + } else { + closed.push(port); + } + }); + return closed; +} + +/** + * What a peer heard over UDP on one port, read against our token: `ours`, + * `silent`, `otherAnswer`, or `notProbed` for a port it could not send to. + * @param {object} udpAnswered - port -> what that port replied, or null, from the peer + * @param {number} port + * @param {string} token + * @returns {string} + */ +function udpReadingOf(udpAnswered, port, token) { + const reply = udpAnswered[port] !== undefined ? udpAnswered[port] : udpAnswered[String(port)]; + if (reply === undefined) return 'notProbed'; + if (reply === null) return 'silent'; + return typeof reply === 'string' && reply.includes(token) ? 'ours' : 'otherAnswer'; +} + +/** + * Each peer whose UDP evidence still stands, with the first port it heard + * nothing on that no peer has proven. + * @param {Map} udpDisagreements - peer -> ports it heard nothing on + * @param {Set} proven + * @returns {Map} + */ +function udpDisputes(udpDisagreements, proven) { + const disputes = new Map(); + udpDisagreements.forEach((ports, peer) => { + const port = ports.find((closed) => !proven.has(closed)); + if (port !== undefined) disputes.set(peer, port); + }); + return disputes; +} + +/** + * Whether every port a peer probes over UDP is proven. + * @param {number[]} portsToTest + * @param {Set} proven + * @returns {boolean} + */ +function everyUdpPortProven(portsToTest, proven) { + return portsToTest.every((port) => port < config.fluxapps.portMin || port > config.fluxapps.portMax || proven.has(port)); +} + /** * To check if app ports are available publicly before installation. * @@ -817,12 +910,21 @@ function portNotOurs(portsToTest, answered, token) { * about the network, and only one of them is evidence. The object returned is * the one published to the harness bus, so the two cannot drift. * + * Behind UPnP each port is mapped for the test under the application's own + * description, the one its installation maps it under: a mapping the router + * keeps after the test - a removal it refused, an entry it will not rewrite - + * is then the application's as it installs, or a stale one of an application + * this node does not hold, for the cleanup. + * * @param {Array} portsToTest Array of ports to test + * @param {string} appName the application the ports are for * @returns {Promise<{ok: boolean, reason: string, port?: number, peers?: string[], * readings?: object, asked?: string[], silent?: boolean}>} */ -async function checkInstallingAppPortAvailable(portsToTest = []) { +async function checkInstallingAppPortAvailable(portsToTest = [], appName = '') { + const description = `Flux_App_${appName}`; const beforeAppInstallTestingServers = []; + const udpTestServers = []; // One secret for this whole test run, published by our own test servers and // never sent to the peer. The peer returns what it read; we compare. const portTestToken = crypto.randomBytes(16).toString('hex'); @@ -869,13 +971,15 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { // now open this port properly and launch listening on it if (firewallActive) { // eslint-disable-next-line no-await-in-loop - await fluxNetworkHelper.allowTestPort(portToTest); + await fluxNetworkHelper.allowTestPort(portToTest, 'tcp'); + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.allowTestPort(portToTest, 'udp'); } if (isUPNP) { // eslint-disable-next-line no-await-in-loop - const upnpMapResult = await upnpService.mapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`); - if (!upnpMapResult) { - throw new Error('Failed to create map UPNP port'); + const upnpMapResult = await upnpService.mapUpnpPort(portToTest, description); + if (!upnpMapResult.mapped) { + throw new Error(`Failed to create map UPNP port: ${upnpMapResult.reason}`); } } const testHttpServer = new fluxHttpTestServer.FluxHttpTestServer(portTestToken); @@ -906,6 +1010,11 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { const error = await listening.catch((err) => err); if (error) throw error; + + const udpTestServer = new fluxUdpTestServer.FluxUdpTestServer(portTestToken); + udpTestServers.push(udpTestServer); + // eslint-disable-next-line no-await-in-loop + await udpTestServer.listen(portToTest); } await serviceHelper.delay(config.fluxapps.portTestPropagationDelayMs); @@ -923,6 +1032,9 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { // it should find is deliberately not here: a peer that never learns it // cannot produce it without actually reaching us. echo: true, + // Over both protocols: UDP is probed once TCP has passed, and what replied + // is handed back. + protocols: ['tcp', 'udp'], }; const stringData = JSON.stringify(data); // eslint-disable-next-line no-await-in-loop @@ -943,6 +1055,22 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { // Peer address -> the port that peer said is not serving this node. Keyed by // peer so that redrawing the same one does not read as a second opinion. const disagreements = new Map(); + // Peer address -> the ports that peer heard nothing of ours on over UDP. + // Kept apart from TCP: a peer that reached every port over TCP and heard + // nothing over UDP is saying something else. + const udpDisagreements = new Map(); + // Our token came back over TCP from every port, from one peer. Proof, so no + // later peer's reading can outweigh it. + let tcpProven = false; + // Ports our token came back from over UDP, from any peer. + const udpProven = new Set(); + // Peer address -> what it heard over UDP on the port it read as not ours. + // A port answered over TCP by something else is a port forwarded elsewhere, + // so what the same peer heard over UDP describes that forward, not whether + // this node receives UDP: it goes with that peer's reading, never into the + // UDP tally. + const udpWhereNotOurs = new Map(); + let sawNoUdpPeer = false; // Every peer already asked, so a redraw is ANOTHER peer rather than another // draw. Without this the picker can hand back the one just asked - which on // a budget of two attempts it often does - and a check counting distinct @@ -977,6 +1105,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { const askingIP = extractIp(randomSocketAddress); const askingIpPort = extractPort(randomSocketAddress); asked.push(randomSocketAddress); + udpTestServers.forEach((udpTestServer) => udpTestServer.allow(askingIP)); // first check against our IP address // eslint-disable-next-line no-await-in-loop @@ -988,6 +1117,7 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { // What this attempt learned about one of our ports, or null when it // learned nothing at all. Only a port recorded here counts as a witness. let reportedPort = null; + let udpClosedPorts = []; if (!resMyAppAvailability) { // The peer never answered us. That is a fact about the peer, not about @@ -1021,9 +1151,9 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { // token coming back settles a port on one peer's say-so, because only // this node could have produced it. log.warn(`checkInstallingAppPortAvailable - ${askingIP} could not reach port ${failedPort}`); - reportedPort = failedPort; + if (!tcpProven) reportedPort = failedPort; } else if (resMyAppAvailability.data.status === 'success') { - const { answered } = resMyAppAvailability.data.data || {}; + const { answered, udpAnswered } = resMyAppAvailability.data.data || {}; if (!answered) { // This peer is on older code: it reached the ports but did not read @@ -1038,13 +1168,31 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { if (notOurs === null) { // Our own token came back. Proof, not report - only this node could - // have produced it - so one peer settles it. + // have produced it - so one peer settles TCP. + tcpProven = true; + } else if (!tcpProven) { + log.info(`checkInstallingAppPortAvailable - ${askingIP} read something other than this node on port ${notOurs}`); + reportedPort = notOurs; + } + + if (udpAnswered && typeof udpAnswered === 'object' && notOurs !== null) { + udpWhereNotOurs.set(askingIP, udpReadingOf(udpAnswered, notOurs, portTestToken)); + } else if (udpAnswered && typeof udpAnswered === 'object') { + udpClosedPorts = udpPortsNotOurs(portsToTest, udpAnswered, portTestToken, udpProven); + if (udpClosedPorts.length) { + log.warn(`checkInstallingAppPortAvailable - ${askingIP} heard nothing of this node over UDP on port(s) ${udpClosedPorts.join(', ')}`); + } + } else { + // A peer on older code: it does not probe UDP. + sawNoUdpPeer = true; + } + + if (tcpProven && everyUdpPortProven(portsToTest, udpProven)) { decision = { ok: true, reason: 'proven', port: null }; + fluxNetworkHelper.setUdpReachability({ reachable: true }); + fluxEventBus.publish('ports:proven', { ...decision, ports: portsToTest, peer: askingIP }); break; } - - log.info(`checkInstallingAppPortAvailable - ${askingIP} read something other than this node on port ${notOurs}`); - reportedPort = notOurs; } else { // An answer in a shape this node does not understand is not an answer. sawSilentPeer = true; @@ -1052,6 +1200,17 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { continue; } + // UDP silence is decided after the loop: a peer that cannot hear UDP + // replies is silent for every node it probes, so every attempt left is + // spent looking for a peer that proves the port before two silent ones + // refuse it. + if (udpClosedPorts.length) udpDisagreements.set(askingIP, udpClosedPorts); + + if (reportedPort === null) { + // eslint-disable-next-line no-continue + continue; + } + disagreements.set(askingIP, reportedPort); const refused = refusedPort(disagreements, PORT_TEST_CORROBORATION); @@ -1076,17 +1235,49 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { port: refused, peers: [...disagreements.keys()], readings: Object.fromEntries(disagreements), + // what each of those peers heard over UDP on the same port, where it probed + udpReadings: Object.fromEntries([...disagreements.keys()] + .filter((peer) => udpWhereNotOurs.has(peer)) + .map((peer) => [peer, udpWhereNotOurs.get(peer)])), }; fluxEventBus.publish('ports:notOurs', decision); } + if (decision === null) { + const disputed = udpDisputes(udpDisagreements, udpProven); + const udpRefused = refusedPort(disputed, PORT_TEST_CORROBORATION); + if (udpRefused !== null) { + const readings = [...disputed.entries()].map(([peer, port]) => `${peer} on port ${port}`); + fluxNetworkHelper.setUdpReachability({ reachable: false, port: udpRefused, peers: [...disputed.keys()] }); + if (udpInstallRefusalOff()) { + log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} does not answer this node over UDP, as heard by ` + + `${readings.join(', ')}; the network policy has the refusal over UDP off, so the install goes ahead`); + } else { + log.warn(`checkInstallingAppPortAvailable - port ${udpRefused} at this address does not answer this node over UDP, as heard by ` + + `${disputed.size} peers (${readings.join(', ')}) and proven by none of the ${asked.length} asked. Installation aborted.`); + decision = { + ok: false, + reason: 'udpUnreachable', + port: udpRefused, + peers: [...disputed.keys()], + readings: Object.fromEntries(disputed), + }; + fluxEventBus.publish('ports:udpUnreachable', decision); + } + } + } + if (decision === null) { // Nothing proved and nothing corroborated. What was not learned does not // refuse an install: this is the check this node made before the token // existed, and it is what stops the first nodes to upgrade refusing // everything while the rest of the network catches up. let reason; - if (disagreements.size) { + if (tcpProven) { + // Proven over TCP; over UDP either no peer that probes it was asked, or + // fewer than two heard nothing. + reason = 'udpUnproven'; + } else if (disagreements.size) { reason = ranOutOfObservers ? 'noOtherObserver' : 'singleWitness'; } else if (ranOutOfObservers) { reason = 'noObserver'; @@ -1096,10 +1287,15 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { reason = 'noneAnswered'; } - const witnesses = [...disagreements.keys()]; - const disputed = disagreements.size ? [...disagreements.values()][0] : null; + const evidence = reason === 'udpUnproven' ? udpDisputes(udpDisagreements, udpProven) : disagreements; + const witnesses = [...evidence.keys()]; + const disputed = evidence.size ? [...evidence.values()][0] : null; - if (witnesses.length) { + if (reason === 'udpUnproven') { + log.warn(`checkInstallingAppPortAvailable - proven over TCP; over UDP ${witnesses.length} peer(s) heard nothing ` + + `(${witnesses.join(', ') || 'none'})${sawNoUdpPeer ? ' and some peers asked do not probe UDP' : ''}; ` + + 'proceeding without proof over UDP'); + } else if (witnesses.length) { log.warn(`checkInstallingAppPortAvailable - ${witnesses.length} peer(s) read a port that was not ours ` + `(${witnesses.join(', ')}) and no other Flux node outside this address could be asked; ` + 'proceeding on reachability alone'); @@ -1127,13 +1323,16 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { for (const portToTest of portsToTest) { if (firewallActive) { // eslint-disable-next-line no-await-in-loop - await fluxNetworkHelper.deleteTestPortRule(portToTest); + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'tcp'); + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'udp'); } if (isUPNP) { // eslint-disable-next-line no-await-in-loop - await upnpService.removeMapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`); + await upnpService.removeMapUpnpPort(portToTest, description); } } + udpTestServers.forEach((udpTestServer) => udpTestServer.close()); // Close all test servers and wait for them to finish await Promise.all( beforeAppInstallTestingServers.map((server) => new Promise((resolve) => { @@ -1154,13 +1353,16 @@ async function checkInstallingAppPortAvailable(portsToTest = []) { for (const portToTest of portsToTest) { if (firewallActive) { // eslint-disable-next-line no-await-in-loop - await fluxNetworkHelper.deleteTestPortRule(portToTest).catch((e) => log.error(e)); + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'tcp').catch((e) => log.error(e)); + // eslint-disable-next-line no-await-in-loop + await fluxNetworkHelper.deleteTestPortRule(portToTest, 'udp').catch((e) => log.error(e)); } if (isUPNP) { // eslint-disable-next-line no-await-in-loop - await upnpService.removeMapUpnpPort(portToTest, `Flux_Prelaunch_App_${portToTest}`).catch((e) => log.error(e)); + await upnpService.removeMapUpnpPort(portToTest, description).catch((e) => log.error(e)); } } + udpTestServers.forEach((udpTestServer) => udpTestServer.close()); // Close all test servers and wait for them to finish await Promise.all( beforeAppInstallTestingServers.map((server) => new Promise((resolve) => { @@ -1299,6 +1501,10 @@ module.exports = { portsInUse, portsInUseApi, portNotOurs, + udpPortsNotOurs, + udpReadingOf, + udpDisputes, + everyUdpPortProven, refusedPort, siblingHoldingPort, checkInstallingAppPortAvailable, diff --git a/ZelBack/src/services/appQuery/appQueryService.js b/ZelBack/src/services/appQuery/appQueryService.js index c7212c1f1d..456db6c165 100644 --- a/ZelBack/src/services/appQuery/appQueryService.js +++ b/ZelBack/src/services/appQuery/appQueryService.js @@ -6,6 +6,7 @@ const dockerService = require('../dockerService'); const registryManager = require('../appDatabase/registryManager'); const appsRuntimeState = require('../appManagement/appsRuntimeState'); const appConstants = require('../utils/appConstants'); +const mountParser = require('../utils/mountParser'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); const { specificationFormatter } = require('../utils/appSpecHelpers'); const fluxCaching = require('../utils/cacheManager'); @@ -16,6 +17,7 @@ const { socketAddressesMatch } = require('../utils/socketAddressUtils'); const networkStateService = require('../networkStateService'); const peerIdentityService = require('../peerIdentityService'); const primaryRoleChanges = require('../appLifecycle/primaryRoleChanges'); +const folderStates = require('../appMonitoring/folderStates'); const syncthingService = require('../syncthingService'); const log = require('../../lib/log'); @@ -47,10 +49,7 @@ async function decryptEnterpriseSpec(spec) { const cache = fluxCaching.default.enterpriseAppDecryptionCache; const cached = cache.get(cacheKey); - if (cached) { - log.info(`Using cached decrypted app for ${spec.name} (${cacheKey})`); - return cached; - } + if (cached) return cached; const failure = decryptFailures.get(cacheKey); if (failure && Date.now() - failure.at < DECRYPT_FAILURE_WINDOW_MS) { @@ -140,16 +139,26 @@ async function decryptEnterpriseApps(apps, options = {}) { return { readable, unreadable, inPlace }; } +/** + * The specifications of the apps installed on this node, from its own table. + * @param {object} [query] Narrows the read, as a filter on the table. + * @returns {Promise} + * @throws {Error} When the table cannot be read. + */ +async function installedAppSpecs(query = {}) { + const appsDatabase = dbHelper.databaseConnection().db(config.database.appslocal.database); + return dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, query, { projection: { _id: 0 } }); +} + /** * The names of the apps installed on this node that the network lists at - * `socketAddress`: what a node that held that address left there. + * `socketAddress`: what a node that held that address left there. Rejects + * when either cannot be read. * @param {string} socketAddress * @returns {Promise} */ async function installedAppsListedAt(socketAddress) { - const installed = await installedApps(); - if (installed.status !== 'success') return []; - const names = new Set(installed.data.map((app) => app.name)); + const names = new Set((await installedAppSpecs()).map((app) => app.name)); return (await registryManager.appsListedAt(socketAddress)).filter((name) => names.has(name)); } @@ -161,9 +170,6 @@ async function installedAppsListedAt(socketAddress) { */ async function installedApps(req, res) { try { - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - let appsQuery = {}; if (req && req.params && req.query) { let { appname } = req.params; @@ -175,11 +181,7 @@ async function installedApps(req, res) { appsQuery = { name: req }; } - const appsProjection = { - projection: { _id: 0 }, - }; - - const apps = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); + const apps = await installedAppSpecs(appsQuery); const dataResponse = messageHelper.createDataMessage(apps); return res ? res.json(dataResponse) : dataResponse; } catch (error) { @@ -381,14 +383,50 @@ async function sendingFolderIds() { } } +/** + * The container names of the g: components of each installed app that has more + * than one, anchor first (mountParser.anchorComponentIdentifier). + * + * Throws when the installed apps cannot be read. + * @returns {Promise} + */ +async function multiComponentElectedApps() { + const installed = await installedApps(); + if (installed.status !== 'success') throw new Error('the installed apps cannot be read'); + const { inPlace } = await decryptEnterpriseApps(installed.data); + return inPlace + .map((app) => mountParser.gComponentIdentifiers(app).map((identifier) => dockerService.getAppIdentifier(identifier))) + .filter((names) => names.length > 1); +} + +/** + * A holdings list with the anchor of every app it holds any g: component of. + * + * A peer on a release that elects one g: component asks only about the anchor, + * so to it a node holding another of the app's g: components - one whose + * anchor has not started yet, has stopped on its own, or failed to start - + * holds nothing, and it starts the app beside it. + * @param {string[]} holding Container names. + * @param {string[][]} apps As multiComponentElectedApps. + * @returns {string[]} + */ +function withAnchors(holding, apps) { + const names = new Set(holding); + apps.forEach((members) => { + if (members.some((name) => names.has(name))) names.add(members[0]); + }); + return [...names]; +} + /** * What this node holds, both ways a peer reads it: `decided` is its own account * (ownHoldings), and `held` adds every component whose folder sends here. A * component in `held` and not in `decided` has a sending folder and nothing * behind it yet: a primary back from a stop, still deciding whether another - * holder took over. + * holder took over. Each list carries the anchor of every app it holds a g: + * component of (withAnchors). * - * Throws when the lock store cannot be read. + * Throws when the lock store or the installed apps cannot be read. * @returns {Promise<{decided: string[], held: string[]}|null>} null while * syncthing cannot say which folders send here */ @@ -396,7 +434,11 @@ async function holdingsAccount() { const sending = await sendingFolderIds(); if (sending === null) return null; const decided = await ownHoldings(); - return { decided, held: [...new Set([...decided, ...sending])] }; + const apps = await multiComponentElectedApps(); + return { + decided: withAnchors(decided, apps), + held: withAnchors([...decided, ...sending], apps), + }; } /** @@ -781,13 +823,23 @@ async function callerIsFluxnode(body) { * The answer is signed over the challenge the request body carries, when it * carries one. * - * @param {object} req Request. - * @param {object} res Response. * `seeding` is where this node stands in deciding which holder seeds each folder at a * cold start (globalState.seedMarks); it carries the same figures as `holding`, so it * goes only where `holding` does. * - * @returns {object} Message carrying { ready, folders, holding, seeding }. + * `notLookedAt` is every receive-only folder of an installed app this node has not yet + * looked at (folderStates). A peer about to seed waits on them, so a holder that has + * not looked is never read as holding nothing. + * + * `heldOut` and `volumeGone` are the receive-only folders this node cannot seed because + * their volume will not mount: under `heldOut` with the volume's image still on the node, + * so what it holds is unknown, and under `volumeGone` with the image gone, so it holds + * nothing. A peer about to seed leaves this node out of the choice, and does not seed + * over a `heldOut` folder unless another holder has data. + * + * @param {object} req Request. + * @param {object} res Response. + * @returns {object} Message carrying { ready, folders, notLookedAt, heldOut, volumeGone, holding, seeding }. */ async function promotedFolderHoldings(req, res) { try { @@ -817,13 +869,21 @@ async function promotedFolderHoldings(req, res) { const authorized = await verificationHelper.verifyPrivilege(Privilege.FLUX_TEAM, authOf(req)); const entitled = authorized === true || await callerIsFluxnode(body); - const held = globalState.folderHoldings; + const ready = ids !== null; + const { + holding, notLookedAt, heldOut, volumeGone, + } = ready ? folderStates.answer(ids) : { + holding: {}, notLookedAt: [], heldOut: [], volumeGone: [], + }; const seal = peerIdentityService.answerSealer(peerIdentityService.AnswerPurpose.PROMOTED_FOLDERS, body); const response = messageHelper.createDataMessage(await seal({ - ready: ids !== null, + ready, folders: ids === null ? [] : [...ids], + notLookedAt, + heldOut, + volumeGone, ...(entitled ? { - holding: held === null ? {} : Object.fromEntries(held), + holding, seeding: Object.fromEntries(globalState.seedMarks), } : {}), })); @@ -840,6 +900,7 @@ async function promotedFolderHoldings(req, res) { } module.exports = { + installedAppSpecs, installedApps, installedAppsListedAt, decryptEnterpriseApps, diff --git a/ZelBack/src/services/appQuery/resourceQueryService.js b/ZelBack/src/services/appQuery/resourceQueryService.js index 3e797fd08c..2ad6938b2b 100644 --- a/ZelBack/src/services/appQuery/resourceQueryService.js +++ b/ZelBack/src/services/appQuery/resourceQueryService.js @@ -75,72 +75,65 @@ async function fluxUsage(req, res) { * of them means this node cannot account for itself. It does not leave the * process: the API halves publish the totals alone. * - * @returns {object} Message carrying {appsCpusLocked, appsRamLocked, appsHddLocked, unreadable} + * @returns {object} Message carrying {appsCpusLocked, appsRamLocked, appsHddLocked, unreadable}. + * Rejects with the original error when the installed apps cannot be read: a + * node that cannot count what it has committed has no totals to give. */ async function appsResources() { log.info('Checking appsResources'); - try { - const dbopen = dbHelper.databaseConnection(); - const appsDatabase = dbopen.db(config.database.appslocal.database); - const appsQuery = {}; - const appsProjection = { projection: { _id: 0 } }; - const stored = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); - - // eslint-disable-next-line global-require - const { decryptEnterpriseApps } = require('./appQueryService'); - const { inPlace: appsResult, unreadable } = await decryptEnterpriseApps(stored, { formatSpecs: false }); - - let appsCpusLocked = 0; - let appsRamLocked = 0; - let appsHddLocked = 0; - const tier = await generalService.nodeTier().catch((error) => log.error(error)); - const hddTier = `hdd${tier}`; - const ramTier = `ram${tier}`; - const cpuTier = `cpu${tier}`; - - // Ensure appsResult is an array - const apps = Array.isArray(appsResult) ? appsResult : []; - apps.forEach((app) => { - if (app.version >= 4) { - app.compose.forEach((component) => { - if (component.tiered && tier) { - appsCpusLocked += serviceHelper.ensureNumber(component[cpuTier] || component.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(component[ramTier] || component.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(component[hddTier] || component.hdd) || 0; - } else { - appsCpusLocked += serviceHelper.ensureNumber(component.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(component.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(component.hdd) || 0; - } - appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap - }); - } else if (app.tiered && tier) { - appsCpusLocked += serviceHelper.ensureNumber(app[cpuTier] || app.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(app[ramTier] || app.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(app[hddTier] || app.hdd) || 0; - appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap - } else { - appsCpusLocked += serviceHelper.ensureNumber(app.cpu) || 0; - appsRamLocked += serviceHelper.ensureNumber(app.ram) || 0; - appsHddLocked += serviceHelper.ensureNumber(app.hdd) || 0; + const dbopen = dbHelper.databaseConnection(); + const appsDatabase = dbopen.db(config.database.appslocal.database); + const appsQuery = {}; + const appsProjection = { projection: { _id: 0 } }; + const stored = await dbHelper.findInDatabase(appsDatabase, appConstants.localAppsInformation, appsQuery, appsProjection); + + // eslint-disable-next-line global-require + const { decryptEnterpriseApps } = require('./appQueryService'); + const { inPlace: appsResult, unreadable } = await decryptEnterpriseApps(stored, { formatSpecs: false }); + + let appsCpusLocked = 0; + let appsRamLocked = 0; + let appsHddLocked = 0; + const tier = await generalService.nodeTier().catch((error) => log.error(error)); + const hddTier = `hdd${tier}`; + const ramTier = `ram${tier}`; + const cpuTier = `cpu${tier}`; + + // Ensure appsResult is an array + const apps = Array.isArray(appsResult) ? appsResult : []; + apps.forEach((app) => { + if (app.version >= 4) { + app.compose.forEach((component) => { + if (component.tiered && tier) { + appsCpusLocked += serviceHelper.ensureNumber(component[cpuTier] || component.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(component[ramTier] || component.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(component[hddTier] || component.hdd) || 0; + } else { + appsCpusLocked += serviceHelper.ensureNumber(component.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(component.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(component.hdd) || 0; + } appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap - } - }); - const appsUsage = { - appsCpusLocked, - appsRamLocked, - appsHddLocked, - unreadable: unreadable.map((app) => app.name), - }; - return messageHelper.createDataMessage(appsUsage); - } catch (error) { - log.error(error); - return messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - } + }); + } else if (app.tiered && tier) { + appsCpusLocked += serviceHelper.ensureNumber(app[cpuTier] || app.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(app[ramTier] || app.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(app[hddTier] || app.hdd) || 0; + appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap + } else { + appsCpusLocked += serviceHelper.ensureNumber(app.cpu) || 0; + appsRamLocked += serviceHelper.ensureNumber(app.ram) || 0; + appsHddLocked += serviceHelper.ensureNumber(app.hdd) || 0; + appsHddLocked += config.fluxapps.hddFileSystemMinimum + config.fluxapps.defaultSwap; // 5gb per component + 2gb swap + } + }); + const appsUsage = { + appsCpusLocked, + appsRamLocked, + appsHddLocked, + unreadable: unreadable.map((app) => app.name), + }; + return messageHelper.createDataMessage(appsUsage); } /** @@ -179,10 +172,12 @@ function unaccountedApps(response) { * @returns {Promise} */ async function appsResourcesApi(req, res) { - const response = await appsResources(); - - if (response.status === 'error') { - res.json(response); + let response; + try { + response = await appsResources(); + } catch (error) { + log.error(error); + res.json(messageHelper.createErrorMessage(error.message || error, error.name, error.code)); return; } diff --git a/ZelBack/src/services/appRequirements/appValidator.js b/ZelBack/src/services/appRequirements/appValidator.js index 933ae02f40..8f8845b9c7 100644 --- a/ZelBack/src/services/appRequirements/appValidator.js +++ b/ZelBack/src/services/appRequirements/appValidator.js @@ -2,12 +2,8 @@ const config = require('config'); const serviceHelper = require('../serviceHelper'); const messageHelper = require('../messageHelper'); const log = require('../../lib/log'); -const generalService = require('../generalService'); -const verificationHelper = require('../verificationHelper'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); -const fluxCommunicationMessagesSender = require('../fluxCommunicationMessagesSender'); const registryManager = require('../appDatabase/registryManager'); -const messageVerifier = require('../appMessaging/messageVerifier'); const signatureVerifier = require('../signatureVerifier'); const fluxStorage = require('../utils/fluxStorage'); const imageManager = require('../appSecurity/imageManager'); @@ -20,13 +16,10 @@ const { } = require('../utils/appConstants'); const { specificationFormatter, findCommonArchitectures } = require('../utils/appUtilities'); const { checkAndDecryptAppSpecs } = require('../utils/enterpriseHelper'); +const { verifyAppNameAndOwner } = require('../utils/appIdentity'); const placementFeasibility = require('../appPlacement/placementFeasibility'); const enterpriseConfig = require('../utils/enterpriseConfig'); const portManager = require('../appNetwork/portManager'); -const { peerManager } = require('../utils/peerState'); -const { Privilege, authOf } = require('../utils/privileges'); - -const isArcane = Boolean(process.env.FLUXOS_PATH); /** * Verify type correctness of application specification @@ -1510,6 +1503,10 @@ async function verifyAppRegistrationParameters(req, res) { appSpecification.version >= 8 && appSpecification.enterprise, ); + if (isEnterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } + // Decrypt enterprise specifications if needed const appSpecDecrypted = await checkAndDecryptAppSpecs( appSpecification, @@ -1586,6 +1583,10 @@ async function validateAppUpdate(appSpecification) { appSpecification.version >= 8 && appSpecification.enterprise, ); + if (isEnterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } + const decryptedSpecs = await checkAndDecryptAppSpecs(appSpecification, { daemonHeight }); const appSpecFormatted = specificationFormatter(decryptedSpecs); @@ -1665,168 +1666,6 @@ async function verifyAppUpdateApi(req, res) { }); } -/** - * Register application globally via API - * @param {object} req - Request object - * @param {object} res - Response object - * @returns {Promise} Registration result - */ -async function registerAppGlobalyApi(req, res) { - let body = ''; - req.on('data', (data) => { - body += data; - }); - req.on('end', async () => { - try { - const authorized = await verificationHelper.verifyPrivilege(Privilege.USER, authOf(req)); - if (!authorized) { - const errMessage = messageHelper.errUnauthorizedMessage(); - res.json(errMessage); - return; - } - - // first check if this node is available for application registration - if (peerManager.outboundCount < config.fluxapps.minOutgoing) { - throw new Error('Sorry, This Flux does not have enough outgoing peers for safe application registration'); - } - if (peerManager.inboundCount < config.fluxapps.minIncoming) { - throw new Error('Sorry, This Flux does not have enough incoming peers for safe application registration'); - } - - const processedBody = serviceHelper.ensureObject(body); - // Note. Actually signature, timestamp is not needed. But we require it only to verify that user indeed has access to the private key of the owner zelid. - // name and port HAVE to be unique for application. Check if they don't exist in global database - // first let's check if all fields are present and have proper format except tiered and tiered specifications and those can be omitted - let { appSpecification, timestamp, signature } = processedBody; - let messageType = processedBody.type; // determines how data is treated in the future - let typeVersion = processedBody.version; // further determines how data is treated in the future - - if (!appSpecification || !timestamp || !signature || !messageType || !typeVersion) { - throw new Error('Incomplete message received. Check if appSpecification, type, version, timestamp and signature are provided.'); - } - - if (messageType !== 'zelappregister' && messageType !== 'fluxappregister') { - throw new Error('Invalid type of message'); - } - - if (typeVersion !== 1) { - throw new Error('Invalid version of message'); - } - - appSpecification = serviceHelper.ensureObject(appSpecification); - timestamp = serviceHelper.ensureNumber(timestamp); - signature = serviceHelper.ensureString(signature); - messageType = serviceHelper.ensureString(messageType); - typeVersion = serviceHelper.ensureNumber(typeVersion); - - const timestampNow = Date.now(); - if (timestamp < timestampNow - 1000 * 3600) { - throw new Error('Message timestamp is over 1 hour old, not valid. Check if your computer clock is synced and restart the registration process.'); - } else if (timestamp > timestampNow + 1000 * 60 * 5) { - throw new Error('Message timestamp from future, not valid. Check if your computer clock is synced and restart the registration process.'); - } - - const syncStatus = daemonServiceMiscRpcs.isDaemonSynced(); - if (!syncStatus.data.synced) { - throw new Error('Daemon not yet synced.'); - } - - const daemonHeight = syncStatus.data.height; - - const appSpecDecrypted = await checkAndDecryptAppSpecs( - appSpecification, - { - daemonHeight, - owner: appSpecification.owner, - }, - ); - - const appSpecFormatted = specificationFormatter(appSpecDecrypted); - - // parameters are now proper format and assigned. Check for their validity, if they are within limits, have propper ports, repotag exists, string lengths, specs are ok - await verifyAppSpecifications(appSpecFormatted, daemonHeight, true); - - if (appSpecFormatted.version === 7 && appSpecFormatted.nodes.length > 0) { - // eslint-disable-next-line no-restricted-syntax - for (const appComponent of appSpecFormatted.compose) { - if (appComponent.secrets) { - // eslint-disable-next-line no-await-in-loop - await imageManager.checkAppSecrets(appSpecFormatted.name, appComponent, appSpecFormatted.owner, true); - } - } - } - - // check if name is not yet registered - await registryManager.checkApplicationRegistrationNameConflicts(appSpecFormatted); - - const isEnterprise = Boolean( - appSpecification.version >= 8 && appSpecification.enterprise, - ); - - const toVerify = isEnterprise - ? specificationFormatter(appSpecification) - : appSpecFormatted; - - // check if zelid owner is correct ( done in message verification ) - // if signature is not correct, then specifications are not correct type or bad message received. Respond with 'Received message is invalid'; - await messageVerifier.verifyAppMessageSignature(messageType, typeVersion, toVerify, timestamp, signature); - - if (isEnterprise) { - appSpecFormatted.contacts = []; - appSpecFormatted.compose = []; - } - - // if all ok, then sha256 hash of entire message = message + timestamp + signature. We are hashing all to have always unique value. - // If hashing just specificiations, if application goes back to previous specifications, it may pose some issues if we have indeed correct state - // We respond with a hash that is supposed to go to transaction. - const message = messageType + typeVersion + JSON.stringify(appSpecFormatted) + timestamp + signature; - const messageHASH = await generalService.messageHash(message); - - // now all is great. Store appSpecFormatted, timestamp, signature and hash in appsTemporaryMessages. with 1 hours expiration time. Broadcast this message to all outgoing connections. - const temporaryAppMessage = { // specification of temp message - type: messageType, - version: typeVersion, - appSpecifications: appSpecFormatted, - hash: messageHASH, - timestamp, - signature, - arcaneSender: isArcane, - }; - await fluxCommunicationMessagesSender.broadcastTemporaryAppMessage(temporaryAppMessage); - // above takes 2-3 seconds - await serviceHelper.delay(1200); // it takes receiving node at least 1 second to process the message. Add 1200 ms mas for processing - // this operations takes 2.5-3.5 seconds and is heavy, message gets verified again. - await messageVerifier.requestAppMessage(messageHASH); // this itself verifies that Peers received our message broadcast AND peers send us the message back. By peers sending the message back we finally store it to our temporary message storage and rebroadcast it again - // request app message is quite slow and from performance testing message will appear roughly 5 seconds after ask - await serviceHelper.delay(1200); // 1200 ms mas for processing - peer sends message back to us - // check temporary message storage - let tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(messageHASH); // Cumulus measurement: after roughly 8 seconds here - for (let i = 0; i < 20; i += 1) { // ask for up to 20 times - 10 seconds. Must have been processed by that time or it failed. Cumulus measurement: Approx 5-6 seconds - if (!tempMessage) { - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(500); - // eslint-disable-next-line no-await-in-loop - tempMessage = await messageVerifier.checkAppTemporaryMessageExistence(messageHASH); - } - } - if (tempMessage && typeof tempMessage === 'object' && !Array.isArray(tempMessage)) { - const responseHash = messageHelper.createDataMessage(tempMessage.hash); - res.json(responseHash); // all ok - return; - } - throw new Error('Unable to register application on the network. Try again later.'); - } catch (error) { - log.warn(error); - const errorResponse = messageHelper.createErrorMessage( - error.message || error, - error.name, - error.code, - ); - res.json(errorResponse); - } - }); -} - module.exports = { verifyTypeCorrectnessOfApp, verifyRestrictionCorrectnessOfApp, @@ -1837,5 +1676,4 @@ module.exports = { verifyAppRegistrationParameters, validateAppUpdate, verifyAppUpdateApi, - registerAppGlobalyApi, }; diff --git a/ZelBack/src/services/appTamperingDetectionService.js b/ZelBack/src/services/appTamperingDetectionService.js index a80a66ff8f..36cb6583d9 100644 --- a/ZelBack/src/services/appTamperingDetectionService.js +++ b/ZelBack/src/services/appTamperingDetectionService.js @@ -268,7 +268,8 @@ function deriveMainAppName(appName) { * Owner/spec-hash attribution for an app from the local installed-apps * database, cached per emitted name. Tries the name as passed first (a real * app name may itself start with 'flux'), then the derived main app name. - * Best-effort: returns null fields when the app cannot be found. + * Null fields when the app cannot be found; rejects when the installed apps + * cannot be read. * @param {string} appName - Name exactly as passed by the emitter * @returns {Promise} */ @@ -276,35 +277,134 @@ async function getAppAttribution(appName) { if (!appName) return null; const cached = appAttributionCache.get(appName); if (cached && Date.now() - cached.cachedAt < ATTRIBUTION_TTL_MS) return cached; + const db = dbHelper.databaseConnection(); + if (!db) throw new Error('no database connection'); + const appsDatabase = db.db(config.database.appslocal.database); + const projection = { projection: { _id: 0, owner: 1, hash: 1 } }; + let app = await dbHelper.findOneInDatabase( + appsDatabase, localAppsInformation, { name: appName }, projection, + ); + if (!app) { + const mainName = deriveMainAppName(appName); + if (mainName !== appName) { + app = await dbHelper.findOneInDatabase( + appsDatabase, localAppsInformation, { name: mainName }, projection, + ); + } + } + const attribution = { + ownerZelid: app?.owner ?? null, + appHash: app?.hash ?? null, + cachedAt: Date.now(), + }; + appAttributionCache.set(appName, attribution); + return attribution; +} + +// The delays before incidents that could not be written are written again, by +// attempt; the last repeats until they are. At most PENDING_INCIDENTS_MAX wait, +// the oldest going first. A retry is idempotent for the score: it sums one +// severity per incident document, whatever its count. +const INCIDENT_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; +const PENDING_INCIDENTS_MAX = 500; +const pendingIncidents = []; +let incidentRetryTimer = null; + +/** + * Writes one observation onto its incident rollup. Rejects when the database + * does not answer. + * @param {object} observation From recordEvent. + * @returns {Promise} + */ +async function writeIncident(observation) { + const { + appName, eventType, details, at, bootId, uptimeSec, + } = observation; + const db = dbHelper.databaseConnection(); + if (!db) throw new Error('no database connection'); + const database = db.db(config.database.local.database); + const incidentKey = `${bootId ?? 'unknown'}:${Math.floor(at.getTime() / INCIDENT_BUCKET_MS)}`; + const [identity, attribution] = await Promise.all([ + getNodeIdentity(), + getAppAttribution(appName), + ]); + const query = { appName, eventType, incidentKey }; + const update = { + $setOnInsert: { + schemaVersion: 1, + appName, + eventType, + incidentKey, + severity: EVENT_SEVERITY[eventType] ?? 0, + bootId, + uptimeSecAtEvent: uptimeSec, + firstSeen: at, + detailsSample: details, + nodeTxid: identity?.nodeTxid ?? null, + nodeOutidx: identity?.nodeOutidx ?? null, + nodeIp: identity?.nodeIp ?? null, + pubkey: identity?.pubkey ?? null, + paymentAddress: identity?.paymentAddress ?? null, + ownerZelid: attribution?.ownerZelid ?? null, + appHash: attribution?.appHash ?? null, + }, + $set: { lastSeen: at }, + $inc: { count: 1 }, + }; + let result; try { - const db = dbHelper.databaseConnection(); - if (!db) return null; - const appsDatabase = db.db(config.database.appslocal.database); - const projection = { projection: { _id: 0, owner: 1, hash: 1 } }; - let app = await dbHelper.findOneInDatabase( - appsDatabase, localAppsInformation, { name: appName }, projection, + result = await dbHelper.findOneAndUpdateInDatabase( + database, tamperingEventsCollection, query, update, { upsert: true }, ); - if (!app) { - const mainName = deriveMainAppName(appName); - if (mainName !== appName) { - app = await dbHelper.findOneInDatabase( - appsDatabase, localAppsInformation, { name: mainName }, projection, - ); - } - } - const attribution = { - ownerZelid: app?.owner ?? null, - appHash: app?.hash ?? null, - cachedAt: Date.now(), - }; - appAttributionCache.set(appName, attribution); - return attribution; } catch (error) { - log.debug(`appTamperingDetection - attribution lookup failed for ${appName}: ${error.message}`); - return null; + // Two concurrent upserts racing on a brand-new incident key can trip + // the unique index; the loser retries once and lands as the increment. + if (error && error.code === 11000) { + result = await dbHelper.findOneAndUpdateInDatabase( + database, tamperingEventsCollection, query, update, { upsert: true }, + ); + } else { + throw error; + } + } + // Insert detection across driver result shapes: v6 returns the pre-image + // doc or null; older shapes return { value, lastErrorObject }. + const inserted = !result || result.value === null || result?.lastErrorObject?.updatedExisting === false; + if (inserted) { + log.info(`appTamperingDetection - recorded ${eventType} for ${appName}`); + } else { + log.debug(`appTamperingDetection - ${eventType} for ${appName} repeated, incident count incremented`); } } +/** + * Writes the pending incidents in order, stopping at the first that fails and + * trying again after the next delay. + * @param {number} attempt How many flushes failed before this one. + * @returns {Promise} + */ +async function flushPendingIncidents(attempt) { + incidentRetryTimer = null; + while (pendingIncidents.length) { + try { + // eslint-disable-next-line no-await-in-loop + await writeIncident(pendingIncidents[0]); + pendingIncidents.shift(); + } catch (error) { + scheduleIncidentRetry(attempt + 1); + return; + } + } + log.info('appTamperingDetection - pending incidents written'); +} + +function scheduleIncidentRetry(attempt) { + if (incidentRetryTimer) return; + const delay = INCIDENT_RETRY_MS[Math.min(attempt, INCIDENT_RETRY_MS.length - 1)]; + incidentRetryTimer = setTimeout(flushPendingIncidents, delay, attempt); + if (incidentRetryTimer.unref) incidentRetryTimer.unref(); +} + /** * Record a tampering event as an incident rollup (schemaVersion 1). * One document per (appName, eventType, incidentKey) where incidentKey is @@ -314,80 +414,37 @@ async function getAppAttribution(appName) { * identity, app attribution and severity so fleet-level consumers can * attribute and weigh it without joining local state. * Documents expire 30 days after lastSeen (TTL index). + * An event that cannot be written waits in memory and is written later + * (INCIDENT_RETRY_MS), under the time it was seen. * @param {string} appName - Application name * @param {string} eventType - One of the EVENT_SEVERITY keys * @param {string} details - Free-text context (stored once per incident) */ async function recordEvent(rawAppName, eventType, details) { + // Stored under the app's own name whatever a caller addressed it by. The + // boot sweep walks docker component identifiers and the reconciler works + // in app names, so without this the same fault on the same app lands in + // two rows and getEvents, which matches the name exactly, finds one of + // them. Idempotent: an app name carries neither the prefix nor the + // component part this strips. + const observation = { + appName: deriveMainAppName(rawAppName), + eventType, + details, + at: new Date(), + bootId: currentBootId, + uptimeSec: Math.round(os.uptime()), + }; try { - const db = dbHelper.databaseConnection(); - if (!db) { - log.warn('appTamperingDetection - DB not available, skipping event'); - return; - } - const database = db.db(config.database.local.database); - const now = new Date(); - const incidentKey = `${currentBootId ?? 'unknown'}:${Math.floor(now.getTime() / INCIDENT_BUCKET_MS)}`; - // Stored under the app's own name whatever a caller addressed it by. The - // boot sweep walks docker component identifiers and the reconciler works - // in app names, so without this the same fault on the same app lands in - // two rows and getEvents, which matches the name exactly, finds one of - // them. Idempotent: an app name carries neither the prefix nor the - // component part this strips. - const appName = deriveMainAppName(rawAppName); - const [identity, attribution] = await Promise.all([ - getNodeIdentity(), - getAppAttribution(appName), - ]); - const query = { appName, eventType, incidentKey }; - const update = { - $setOnInsert: { - schemaVersion: 1, - appName, - eventType, - incidentKey, - severity: EVENT_SEVERITY[eventType] ?? 0, - bootId: currentBootId, - uptimeSecAtEvent: Math.round(os.uptime()), - firstSeen: now, - detailsSample: details, - nodeTxid: identity?.nodeTxid ?? null, - nodeOutidx: identity?.nodeOutidx ?? null, - nodeIp: identity?.nodeIp ?? null, - pubkey: identity?.pubkey ?? null, - paymentAddress: identity?.paymentAddress ?? null, - ownerZelid: attribution?.ownerZelid ?? null, - appHash: attribution?.appHash ?? null, - }, - $set: { lastSeen: now }, - $inc: { count: 1 }, - }; - let result; - try { - result = await dbHelper.findOneAndUpdateInDatabase( - database, tamperingEventsCollection, query, update, { upsert: true }, - ); - } catch (error) { - // Two concurrent upserts racing on a brand-new incident key can trip - // the unique index; the loser retries once and lands as the increment. - if (error && error.code === 11000) { - result = await dbHelper.findOneAndUpdateInDatabase( - database, tamperingEventsCollection, query, update, { upsert: true }, - ); - } else { - throw error; - } - } - // Insert detection across driver result shapes: v6 returns the pre-image - // doc or null; older shapes return { value, lastErrorObject }. - const inserted = !result || result.value === null || result?.lastErrorObject?.updatedExisting === false; - if (inserted) { - log.info(`appTamperingDetection - recorded ${eventType} for ${appName}`); - } else { - log.debug(`appTamperingDetection - ${eventType} for ${appName} repeated, incident count incremented`); - } + await writeIncident(observation); } catch (error) { - log.error(`appTamperingDetection - failed to record event: ${error.message}`); + log.error(`appTamperingDetection - failed to record event, written again later: ${error.message}`); + if (pendingIncidents.length >= PENDING_INCIDENTS_MAX) { + const dropped = pendingIncidents.shift(); + log.warn(`appTamperingDetection - ${PENDING_INCIDENTS_MAX} incidents wait to be written, dropping ${dropped.eventType} for ${dropped.appName}`); + } + pendingIncidents.push(observation); + scheduleIncidentRetry(0); } } @@ -490,20 +547,43 @@ async function checkNodeReboot() { currentBootId = bootId; const now = new Date(); - const previous = await dbHelper.findOneInDatabase( - database, nodeStartupTrackerCollection, { _id: STARTUP_MARKER_KEY }, + // The machine's boot moment (uptime-derived): cadence consumers want the + // boot period itself, free of FluxOS's variable start lag. + const bootedAt = new Date(now.getTime() - Math.round(os.uptime() * 1000)); + await recordBootUntilWritten(now, bootedAt); + } catch (error) { + log.error(`appTamperingDetection - checkNodeReboot failed: ${error.message}`); + } +} + +// The delays before a boot that could not be recorded is recorded again, by +// attempt; the last repeats until it is. +const BOOT_RECORD_RETRY_MS = [30 * 1000, 60 * 1000, 5 * 60 * 1000, 15 * 60 * 1000]; + +/** + * Records the boot this process runs in: in the boot history the first time + * its boot_id is seen there, and as the startup marker. Repeating it records + * nothing twice. + * @param {Date} now When FluxOS first saw this boot. + * @param {Date} bootedAt When the machine booted. + * @returns {Promise} + */ +async function recordBoot(now, bootedAt) { + const database = dbHelper.databaseConnection().db(config.database.local.database); + const previous = await dbHelper.findOneInDatabase( + database, nodeStartupTrackerCollection, { _id: STARTUP_MARKER_KEY }, + ); + const previousBootId = previous && previous.bootId ? previous.bootId : null; + + if (previousBootId !== currentBootId) { + // First sighting of this boot_id: genuine reboot, first run, or upgrade + // from the pre-boot_id marker. All three land in the boot history; a + // same-boot_id process restart does not, keeping the history a record + // of machine boots rather than FluxOS restarts. + const history = await dbHelper.findOneInDatabase( + database, nodeStartupTrackerCollection, { _id: BOOT_HISTORY_KEY }, ); - const previousBootId = previous && previous.bootId ? previous.bootId : null; - - if (previousBootId !== currentBootId) { - // First sighting of this boot_id: genuine reboot, first run, or upgrade - // from the pre-boot_id marker. All three land in the boot history; a - // same-boot_id process restart does not, keeping the history a record - // of machine boots rather than FluxOS restarts. `bootedAt` is the - // machine's boot moment (uptime-derived) — cadence consumers want the - // boot period itself, free of FluxOS's variable start lag; `at` is when - // FluxOS first saw the boot. - const bootedAt = new Date(now.getTime() - Math.round(os.uptime() * 1000)); + if (!(history?.boots ?? []).some((boot) => boot.bootId === currentBootId)) { await dbHelper.findOneAndUpdateInDatabase( database, nodeStartupTrackerCollection, @@ -511,20 +591,36 @@ async function checkNodeReboot() { { $push: { boots: { $each: [{ bootId: currentBootId, bootedAt, at: now }], $slice: -BOOT_HISTORY_MAX } } }, { upsert: true }, ); - if (previousBootId) { - log.info(`appTamperingDetection - machine reboot: boot_id ${previousBootId.slice(0, 8)} -> ${currentBootId.slice(0, 8)}`); - } } + if (previousBootId) { + log.info(`appTamperingDetection - machine reboot: boot_id ${previousBootId.slice(0, 8)} -> ${currentBootId.slice(0, 8)}`); + } + } - await dbHelper.findOneAndUpdateInDatabase( - database, - nodeStartupTrackerCollection, - { _id: STARTUP_MARKER_KEY }, - { $set: { at: now, bootId: currentBootId } }, - { upsert: true }, - ); + await dbHelper.findOneAndUpdateInDatabase( + database, + nodeStartupTrackerCollection, + { _id: STARTUP_MARKER_KEY }, + { $set: { at: now, bootId: currentBootId } }, + { upsert: true }, + ); +} + +/** + * recordBoot, run again after BOOT_RECORD_RETRY_MS until it succeeds. + * @param {Date} now + * @param {Date} bootedAt + * @param {number} [attempt] + * @returns {Promise} + */ +async function recordBootUntilWritten(now, bootedAt, attempt = 0) { + try { + await recordBoot(now, bootedAt); } catch (error) { - log.error(`appTamperingDetection - checkNodeReboot failed: ${error.message}`); + const delay = BOOT_RECORD_RETRY_MS[Math.min(attempt, BOOT_RECORD_RETRY_MS.length - 1)]; + log.error(`appTamperingDetection - this boot could not be recorded, retrying in ${Math.round(delay / 1000)}s: ${error.message}`); + const timer = setTimeout(recordBootUntilWritten, delay, now, bootedAt, attempt + 1); + if (timer.unref) timer.unref(); } } @@ -542,8 +638,7 @@ async function checkNodeReboot() { * unique index the upsert relies on is already in place. * * Low-stakes local telemetry on a 30-day TTL, so a failure logs and is left for - * the next boot rather than wedging startup - the disposition its owner chooses, - * the same one appsRuntimeState.prepareCollection takes. + * the next boot rather than wedging startup. */ async function prepareIncidentRollup() { try { @@ -582,10 +677,12 @@ async function prepareIncidentRollup() { }; delete merged._id; log.warn(`appTamperingDetection - merged ${group.length} duplicate incident rollups for ${query.appName}/${query.eventType} (count=${merged.count})`); + // The merge lands on one rollup before the others go, so a failure part way + // leaves duplicates for the next boot to merge, never the incident lost. // eslint-disable-next-line no-await-in-loop - await dbHelper.removeDocumentsFromCollection(database, tamperingEventsCollection, query); + await dbHelper.updateOneInDatabase(database, tamperingEventsCollection, { _id: newest._id }, { $set: merged }); // eslint-disable-next-line no-await-in-loop - await dbHelper.updateOneInDatabase(database, tamperingEventsCollection, query, { $set: merged }, { upsert: true }); + await dbHelper.removeDocumentsFromCollection(database, tamperingEventsCollection, { ...query, _id: { $ne: newest._id } }); } await database.collection(tamperingEventsCollection).createIndex( diff --git a/ZelBack/src/services/benchmarkService.js b/ZelBack/src/services/benchmarkService.js index e9303d5e23..c9bdb320a2 100644 --- a/ZelBack/src/services/benchmarkService.js +++ b/ZelBack/src/services/benchmarkService.js @@ -12,6 +12,7 @@ const upnpService = require('./upnpService'); const fluxRpc = require('./utils/fluxRpc'); const dbHelper = require('./dbHelper'); const { Privilege, authOf } = require('./utils/privileges'); +const { isValidAppNameAndOwner } = require('./utils/appIdentity'); // eslint-disable-next-line no-unused-vars const isArcane = Boolean(process.env.FLUXOS_PATH); @@ -279,44 +280,67 @@ async function signFluxTransactionPost(req, res) { }); } +/** + * Whether a request names a well-formed app and owner. + * @param {string} message JSON object with `appName` and `fluxID`. + * @returns {boolean} + */ +function isValidAppRequest(message) { + let request; + try { + request = JSON.parse(message); + } catch { + return false; + } + if (!request || typeof request !== 'object') return false; + + return isValidAppNameAndOwner(request.appName, request.fluxID); +} + +/** + * Ask FluxBench to handle an app request. A request without a valid app + * name and owner is refused here and never sent. + * @param {string} rpc FluxBench RPC. + * @param {string} message JSON object with `appName` and `fluxID`. + * @returns {Promise} Message. + */ +async function executeAppCall(rpc, message) { + if (!isValidAppRequest(message)) { + return messageHelper.createErrorMessage('Invalid app name or owner'); + } + return executeCall(rpc, [message]); +} + /** * Ask FLuxBench to decrypt message - * @param {object} message message object with information to be decrypted. + * @param {string} message JSON object with information to be decrypted. */ async function decryptMessage(message) { - const rpccall = 'decryptmessage'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('decryptmessage', message); } /** * Ask FLuxBench to decrypt rsa message - * @param {object} message message object with information to be decrypted. + * @param {string} message JSON object with information to be decrypted. */ async function decryptRSAMessage(message) { - const rpccall = 'decryptrsamessage'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('decryptrsamessage', message); } /** * Ask FLuxBench to encrypt message - * @param {object} message message object with information to be decrypted. + * @param {string} message JSON object with information to be encrypted. */ async function encryptMessage(message) { - const rpccall = 'encryptmessage'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('encryptmessage', message); } /** * Ask FLuxBench to get public key to encrypt enterprise content - * @param {object} message message object with the key. + * @param {string} message JSON object with the app and its owner. */ async function getPublicKey(message) { - const rpccall = 'getpublickey'; - const rpcparameters = [message]; - return executeCall(rpccall, rpcparameters); + return executeAppCall('getpublickey', message); } // == Control == diff --git a/ZelBack/src/services/databaseHealthService.js b/ZelBack/src/services/databaseHealthService.js new file mode 100644 index 0000000000..736118ccc2 --- /dev/null +++ b/ZelBack/src/services/databaseHealthService.js @@ -0,0 +1,130 @@ +/** + * Hold this node back from new placements while its database is timing out + * FluxOS's calls. + * + * WHAT COUNTS. A round trip that ran out: a pooled connection closed by its own + * socket timeout, or a wait for a pooled connection that ran out. Both are read + * from the clients' pool events, which the driver emits for every call whatever + * code made it. A read is retried once, so one stuck read is two round trips. + * + * WHAT DOES NOT. A connection closed because the server went away. The driver + * clears the pool first, and every connection created before the clear then + * closes with the same reason as a timeout. Connection ids only increase, so + * the newest id at the last clear separates the two. + * + * The hold stops new installs and nothing else. Heavy load also makes calls + * time out, so the apps the node already runs are left alone. + */ + +const config = require('config'); +const log = require('../lib/log'); +const fluxNetworkHelper = require('./fluxNetworkHelper'); + +const OWNER = fluxNetworkHelper.PlacementHoldOwner.DATABASE_SUSPECT; +const SUSPECT_ROUND_TRIPS = config.database.suspectRoundTrips; +const SUSPECT_WINDOW_MS = config.database.suspectWindowMs; + +/** + * When each recent round trip ran out, newest last, never more than + * SUSPECT_ROUND_TRIPS of them: the only question asked of it is whether that + * many fall inside the window. + * @type {number[]} + */ +let timedOut = []; +/** Set while the hold stands; it releases the hold when it fires. */ +let releaseTimer = null; + +function monotonicMs() { + return Number(process.hrtime.bigint() / 1000000n); +} + +/** Every round trip it counted is a full window old by now, so none is left to count. */ +function release() { + releaseTimer = null; + fluxNetworkHelper.clearPlacementHold(OWNER); +} + +/** + * Count one round trip that ran out. The hold is set when SUSPECT_ROUND_TRIPS + * fall within SUSPECT_WINDOW_MS, and released SUSPECT_WINDOW_MS after the last. + * @param {string} what Which client, and how it ran out. + */ +function recordTimeout(what) { + const now = monotonicMs(); + timedOut = [...timedOut, now].filter((at) => now - at < SUSPECT_WINDOW_MS).slice(-SUSPECT_ROUND_TRIPS); + log.warn(`databaseHealth - a database round trip ran out (${what})`); + const held = releaseTimer !== null; + if (!held && timedOut.length < SUSPECT_ROUND_TRIPS) return; + if (!held) { + fluxNetworkHelper.setPlacementHold( + OWNER, + `Database calls are timing out: ${SUSPECT_ROUND_TRIPS} round trips ran out within ${SUSPECT_WINDOW_MS / 60000} minutes`, + ); + } + clearTimeout(releaseTimer); + releaseTimer = setTimeout(release, SUSPECT_WINDOW_MS); +} + +/** The pool events of one client, and which of its closes were timeouts. */ +class PoolTimeouts { + /** + * @param {string} name Names the client in the log. + */ + constructor(name) { + this.name = name; + this.newestCreated = 0; + this.newestBeforeClear = 0; + } + + onCreated(event) { + this.newestCreated = Math.max(this.newestCreated, event.connectionId); + } + + onCleared() { + this.newestBeforeClear = this.newestCreated; + } + + onClosed(event) { + if (event.reason !== 'error' || event.connectionId <= this.newestBeforeClear) return; + recordTimeout(`${this.name} client, connection ${event.connectionId} timed out`); + } + + onCheckOutFailed(event) { + if (event.reason !== 'timeout') return; + recordTimeout(`${this.name} client, no pooled connection within the timeout`); + } +} + +/** + * Count the round trips that run out on this client. + * @param {import('mongodb').MongoClient|null} client + * @param {string} name Names the client in the log. + */ +function watch(client, name) { + if (!client) return; + const pool = new PoolTimeouts(name); + client.on('connectionCreated', pool.onCreated.bind(pool)); + client.on('connectionPoolCleared', pool.onCleared.bind(pool)); + client.on('connectionClosed', pool.onClosed.bind(pool)); + client.on('connectionCheckOutFailed', pool.onCheckOutFailed.bind(pool)); +} + +/** + * @returns {boolean} True while the database holds the node back. + */ +function isSuspect() { + return releaseTimer !== null; +} + +/** Forget every count and release the hold. */ +function stop() { + clearTimeout(releaseTimer); + timedOut = []; + release(); +} + +module.exports = { + watch, + isSuspect, + stop, +}; diff --git a/ZelBack/src/services/dbHelper.js b/ZelBack/src/services/dbHelper.js index ff8be7fc53..8c1346522e 100644 --- a/ZelBack/src/services/dbHelper.js +++ b/ZelBack/src/services/dbHelper.js @@ -19,6 +19,23 @@ const mongoUrl = `mongodb://${config.database.url}:${config.database.port}/`; */ let openDBConnection = null; +// The client for the named long operations: index builds, the explorer +// restore's deletes and the registry rebuild. One of its round trips may take +// up to config.database.maintenanceCallTimeoutMs; every other call uses the +// main client, bounded by config.database.callTimeoutMs. +function maintenanceSettings() { + return { + maxPoolSize: 4, + socketTimeoutMS: config.database.maintenanceCallTimeoutMs, + waitQueueTimeoutMS: config.database.maintenanceCallTimeoutMs, + }; +} + +/** + * @type {mongodb.MongoClient} + */ +let maintenanceDBConnection = null; + /** * Cached MongoDB server version, populated once per connection. * @type {string | null} @@ -34,29 +51,73 @@ function databaseConnection() { return openDBConnection; } +/** + * The maintenance client, if it was initiated before, otherwise null. + * @returns {mongodb.MongoClient | null} + */ +function maintenanceConnection() { + return maintenanceDBConnection; +} + +/** + * `database` on the maintenance client: the same database, whose round trips + * may take up to config.database.maintenanceCallTimeoutMs. `database` itself while the + * maintenance client is not connected. + * @param {mongodb.Db} database + * @returns {mongodb.Db} + */ +function onMaintenanceClient(database) { + return maintenanceDBConnection ? maintenanceDBConnection.db(database.databaseName) : database; +} + +/** + * Whether an error came from the database rather than from the work around it: + * a timeout, a pool wait, an unreachable server, or a refused command. A caller + * deciding whether a failure says anything about an app reads this - a database + * that did not answer says nothing about the app. + * @param {*} error + * @returns {boolean} + */ +function isDatabaseError(error) { + return error instanceof mongodb.MongoError; +} + /** * Initiates connection with the database. * * @param {string} [url] + * @param {object} [settings] Client options over the main client's. * * @returns {Promise} */ -async function connectMongoDb(url) { +async function connectMongoDb(url, settings = {}) { const connectUrl = url || mongoUrl; const mongoSettings = { maxPoolSize: 100, + // One round trip, and the wait for a pooled connection, end in an error + // after callTimeoutMs instead of waiting for ever. A read is retried once, so + // it fails after two; a write is not, and its outcome is then unknown. + socketTimeoutMS: config.database.callTimeoutMs, + waitQueueTimeoutMS: config.database.callTimeoutMs, + ...settings, }; const client = await MongoClient.connect(connectUrl, mongoSettings); return client; } /** - * Initiates default db connection. + * Initiates the default db connection and the maintenance one, both or + * neither. * @returns true */ async function initiateDB() { if (!openDBConnection) { - openDBConnection = await connectMongoDb(); + const main = await connectMongoDb(); + maintenanceDBConnection = await connectMongoDb(undefined, maintenanceSettings()).catch(async (error) => { + await main.close(); + throw error; + }); + openDBConnection = main; // Read the server version once, on the initial connect. It is informational // and the getter swallows its own errors, so this cannot fail the connect. await getMongoDbVersion(); @@ -123,6 +184,168 @@ async function closeDbConnection() { openDBConnection = null; mongoDbVersion = null; } + if (maintenanceDBConnection) { + await maintenanceDBConnection.close(); + maintenanceDBConnection = null; + } +} + +/** + * Remove rows that duplicate a would-be-unique key, keeping the newest of each. + * + * A recovery strategy for ensureIndex: when a unique build fails because the + * collection already holds rows that violate it, this makes the data conform to + * the invariant the index DECLARES - it deletes duplicates on the key the index + * says must be unique, which is enforcing a contract rather than losing data. + * Only safe where the key IS the row's identity, so it is passed in per build by + * the caller that knows the collection, never applied by default. A rollup whose + * duplicates must be summed rather than dropped (the tampering incident count) + * belongs to its owning service instead - see the note on ensureIndex. + * + * Keeps the newest per group (rows sort by _id, which is time-ordered), honours + * the index's partialFilterExpression so it only touches rows the index covers, + * and returns how many it removed. + * + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key, e.g. { hash: 1 } + * @param {object} options - the index options (read for partialFilterExpression) + * @returns {Promise} rows removed + */ +async function dedupeByKey(collection, spec, options = {}) { + const groupId = {}; + Object.keys(spec).forEach((key, i) => { groupId[`k${i}`] = `$${key}`; }); + // Held in memory deliberately, and measured rather than assumed: run against a + // live node's collection unioned with itself until every key appeared 16 times + // - 1,030,208 rows, the duplicate state this exists to repair - it finished in + // under 2s without spilling. The $sort adds nothing on top while it stays an + // index walk on _id, which it is for a spec with no partialFilterExpression; + // the first partial index to use this wants re-measuring, because the $match + // ahead of the sort is what would make the sort blocking. allowDiskUse is not + // set: it would take a mongo below 6.0, where the cap errors instead of + // spilling, and the network floor is moving past that. + // + // ids[0] rather than $max: $group does not document that it carries a + // preceding sort into an accumulator, and that non-guarantee is about results + // merged from several sources - this is one standalone mongod. Checked against + // 64,388 real duplicate groups, ids[0] was the newest in all 64,388. + const pipeline = [ + ...(options.partialFilterExpression ? [{ $match: options.partialFilterExpression }] : []), + { $sort: { _id: -1 } }, + { $group: { _id: groupId, ids: { $push: '$_id' } } }, + { $match: { 'ids.1': { $exists: true } } }, + ]; + const groups = await collection.aggregate(pipeline).toArray(); + const toRemove = groups.flatMap((group) => group.ids.slice(1)); + if (!toRemove.length) return 0; + await collection.deleteMany({ _id: { $in: toRemove } }); + return toRemove.length; +} + +/** + * Assert one index, healing the failures that are recoverable. + * + * - a pre-existing index that conflicts is dropped and recreated: the index on + * the same KEY when its options differ (IndexOptionsConflict), the index of + * the same NAME when its key differs (IndexKeySpecsConflict); + * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` + * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the + * index rather than running degraded without it. Every build goes this way, + * the one after a conflicting index is dropped included (buildIndex); + * - anything else rethrows. + * + * The rethrow is deliberate. Index setup runs in serviceManager's + * prepareDatabaseSchema, which boot runs again every 15 s until it succeeds: a TRANSIENT + * failure (mongo mid-election, a slow-disk blip) heals on the next pass instead + * of being skipped until the next reboot, + * and a genuinely UNRECOVERABLE database wedges loudly - which is correct, since + * a node whose DB cannot hold its schema cannot serve apps and appremove would + * not rescue it. The realistic wedge that finding motivated - a unique index + * over rows that already violate it - is repaired above, not hidden. + * + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key + * @param {object} [options] - the index options + * @param {(collection: object, spec: object, options: object) => Promise} [recover] + * run when a unique build is blocked by existing duplicate rows + */ +/** + * Builds one index; a unique build blocked by rows that already violate it runs + * `recover` and builds again. + * @param {object} collection - a mongo collection handle + * @param {object} spec - the index key + * @param {object} options - the index options + * @param {Function|null} recover + * @returns {Promise} + */ +async function buildIndex(collection, spec, options, recover) { + try { + await collection.createIndex(spec, options); + } catch (err) { + const duplicate = err && (err.code === 11000 || err.codeName === 'DuplicateKey'); + if (!duplicate || !recover) throw err; + const removed = await recover(collection, spec, options); + log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); + await collection.createIndex(spec, options); + } +} + +async function ensureIndex(collection, spec, options = {}, recover = null) { + try { + await buildIndex(collection, spec, options, recover); + } catch (err) { + const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); + if (!conflict) throw err; + const specKeys = JSON.stringify(spec); + const indexes = await collection.listIndexes().toArray(); + const match = err.codeName === 'IndexKeySpecsConflict' + ? indexes.find((idx) => idx.name === options.name) + : indexes.find((idx) => JSON.stringify(idx.key) === specKeys); + if (match?.name) { + log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); + await collection.dropIndex(match.name); + } + await buildIndex(collection, spec, options, recover); + } +} + +/** + * Assert every index one collection needs, in a single command. + * + * mongo's index build protocol has a fixed cost per BUILD - register, start, + * scan, wait for commit quorum, commit, log - and it does not care that the + * collection is empty. A node asserting its schema one index at a time pays + * that cost 34 times over 14 collections; createIndexes pays it once per + * collection. A node boots faster for it, and the integration harness, where + * ten nodes share one mongod and every database is new, feels it ten times over + * (measured: 938 concurrent builds per ten-node fleet). + * + * The batch is the fast path, not the only one. ensureIndex heals two failures + * that need to be attributed to a single index - an options conflict it drops + * and rebuilds, and a unique build blocked by duplicate rows it repairs through + * the caller's strategy - and a batch rejection does not say which member + * failed. So any error falls back to asserting them one at a time. + * + * @param {object} collection - a mongo collection handle + * @param {Array} specs - `{ key, ...indexOptions, recover }` per index, + * where `recover` is ours and never reaches mongo + */ +async function ensureIndexes(collection, specs) { + try { + await collection.createIndexes(specs.map((spec) => { + // `recover` is a FluxOS concern; mongo is handed the index model alone + const model = { ...spec }; + delete model.recover; + return model; + })); + return; + } catch (error) { + log.warn(`ensureIndexes - batch of ${specs.length} on ${collection.collectionName} failed (${error.codeName || error.message}); asserting one at a time`); + } + // eslint-disable-next-line no-restricted-syntax + for (const { key, recover = null, ...options } of specs) { + // eslint-disable-next-line no-await-in-loop + await ensureIndex(collection, key, options, recover); + } } /** @@ -514,14 +737,28 @@ async function updateValueSatInAppsMessages(brokenHashes, hashMap) { } } -async function repairNanInAppsMessagesDb() { - const brokenHashes = await findValueSatNanInAppsMessages(); - - if (!brokenHashes.length) return; +// The marker a completed valueSat repair leaves in the node startup tracker. +const VALUESAT_REPAIR_MARKER = 'appsMessagesValueSatRepaired'; - const hashMap = await findValueSatInAppsHashes(); +/** + * Gives every app message stored with a NaN valueSat the value its payment + * record holds. Runs once: a completed repair leaves a marker, and later calls + * return without scanning, unless `force` is set. + * @param {object} [options] + * @param {boolean} [options.force] Repair whether or not the marker is there. + * @returns {Promise} + */ +async function repairNanInAppsMessagesDb({ force = false } = {}) { + const local = databaseConnection().db(config.database.local.database); + const tracker = config.database.local.collections.nodeStartupTracker; + if (!force && await findOneInDatabase(local, tracker, { _id: VALUESAT_REPAIR_MARKER })) return; - await updateValueSatInAppsMessages(brokenHashes, hashMap); + const brokenHashes = await findValueSatNanInAppsMessages(); + if (brokenHashes.length) { + const hashMap = await findValueSatInAppsHashes(); + await updateValueSatInAppsMessages(brokenHashes, hashMap); + } + await updateOneInDatabase(local, tracker, { _id: VALUESAT_REPAIR_MARKER }, { $set: { repairedAt: new Date() } }, { upsert: true }); } /** @@ -695,13 +932,14 @@ async function liveAppRow(appsGlobalDb, appsMessagesCol, daemonDb, name, scanned /** * Whether the registry holds a different number of live apps than the messages give: each name's * newest message, and for a name the name rule decides (appNamesTheNameRuleDecides), its message - * in force (liveAppRow), as the rebuild takes them. + * in force (liveAppRow), as the rebuild takes them. Null when the messages give no live app, so + * there is nothing to validate against. Rejects when the database does not answer. * @param {mongodb.Db} appsGlobalDb * @param {string} appsMessagesCol mongo collection name * @param {string} appsInformationCol mongo collection name * @param {number} scannedHeight * @param {mongodb.Db} daemonDb the payment records' database - * @returns {Promise} + * @returns {Promise} */ async function isReindexAppsInformationRequired( appsGlobalDb, @@ -761,61 +999,57 @@ async function isReindexAppsInformationRequired( }, ]; - try { - await ensureNewestAppMessageIndex(appsGlobalDb.collection(appsMessagesCol)); + // The index build and the $group run on the maintenance client. + await ensureNewestAppMessageIndex(onMaintenanceClient(appsGlobalDb).collection(appsMessagesCol)); - const messagesCursor = await aggregateInDatabase( - appsGlobalDb, - appsMessagesCol, - appsMessagesPipeline, - { returnArray: false }, - ); - const informationCursor = await aggregateInDatabase( - appsGlobalDb, - appsInformationCol, - appsInformationPipeline, - { returnArray: false }, - ); + const messagesCursor = await aggregateInDatabase( + onMaintenanceClient(appsGlobalDb), + appsMessagesCol, + appsMessagesPipeline, + { returnArray: false }, + ); + const informationCursor = await aggregateInDatabase( + appsGlobalDb, + appsInformationCol, + appsInformationPipeline, + { returnArray: false }, + ); - const newestLive = await messagesCursor.next(); - const appsFromInformation = await informationCursor.next(); - const appsFromMessagesCount = (newestLive?.count ?? 0) + decidedLive; + const newestLive = await messagesCursor.next(); + const appsFromInformation = await informationCursor.next(); + const appsFromMessagesCount = (newestLive?.count ?? 0) + decidedLive; - if (!appsFromMessagesCount) { - log.warn('No apps from apps messages found, unable to validate apps information'); - return false; - } - - if (!appsFromInformation) { - log.info('No apps information apps found, reindexing colleciton'); - return true; - } + if (!appsFromMessagesCount) { + log.warn('No apps from apps messages found, unable to validate apps information'); + return null; + } - log.info( - `Apps reindex validation. Found ${appsFromMessagesCount} apps from appsMessages.` - + ` Found ${appsFromInformation.count} apps from appsInformation`, - ); + if (!appsFromInformation) { + log.info('No apps information apps found, reindexing colleciton'); + return true; + } - if (appsFromMessagesCount !== appsFromInformation.count) { - return true; - } + log.info( + `Apps reindex validation. Found ${appsFromMessagesCount} apps from appsMessages.` + + ` Found ${appsFromInformation.count} apps from appsInformation`, + ); - // Detect ghost flat fields on v4+ specs caused by $set accumulating - // fields from prior spec versions. Fixed by replaceOne in registryManager. - const ghostCount = await countInDatabase(appsGlobalDb, appsInformationCol, { - version: { $gte: 4 }, - repotag: { $exists: true }, - }); - if (ghostCount > 0) { - log.info(`Found ${ghostCount} v4+ specs with ghost fields from prior versions, reindex required`); - return true; - } + if (appsFromMessagesCount !== appsFromInformation.count) { + return true; + } - return false; - } catch (err) { - log.error(`isReindexAppsInformationRequired - Mongodb Error: ${err}`); - return false; + // Detect ghost flat fields on v4+ specs caused by $set accumulating + // fields from prior spec versions. Fixed by replaceOne in registryManager. + const ghostCount = await countInDatabase(appsGlobalDb, appsInformationCol, { + version: { $gte: 4 }, + repotag: { $exists: true }, + }); + if (ghostCount > 0) { + log.info(`Found ${ghostCount} v4+ specs with ghost fields from prior versions, reindex required`); + return true; } + + return false; } /** @@ -913,7 +1147,9 @@ async function reindexGlobalAppsInformation( if (error.message !== 'ns not found') throw error; }); - const infoCol = appsGlobalDb.collection(stagingCol); + // The index builds and the $group below run on the maintenance client. + const longGlobalDb = onMaintenanceClient(appsGlobalDb); + const infoCol = longGlobalDb.collection(stagingCol); await infoCol.createIndexes([ { key: { name: 1 }, name: 'query for getting zelapp based on zelapp specs name' }, { key: { owner: 1 }, name: 'query for getting zelapp based on zelapp specs owner' }, @@ -921,7 +1157,7 @@ async function reindexGlobalAppsInformation( { key: { height: 1 }, name: 'query for getting zelapp based on last height update' }, { key: { hash: 1 }, name: 'query for getting zelapp based on last hash' }, ]); - await ensureNewestAppMessageIndex(appsGlobalDb.collection(globalAppsMessagesCol)); + await ensureNewestAppMessageIndex(longGlobalDb.collection(globalAppsMessagesCol)); const pipeline = [ { $sort: NEWEST_APP_MESSAGE_SORT }, @@ -958,7 +1194,7 @@ async function reindexGlobalAppsInformation( ]; const resultCursor = await aggregateInDatabase( - appsGlobalDb, + longGlobalDb, globalAppsMessagesCol, pipeline, { returnArray: false }, @@ -1004,8 +1240,9 @@ async function reindexGlobalAppsInformation( /** * Verifies the app count based on an aggregation from appsmessages and compares it to the * app count in appsinformation. If they differ - the appsinformation collection is dropped and - * rebuilt from the appsmessages. The entire process takes about 500-700ms. - * @returns {Promise<{validated: boolean, reindexed: boolean}>} + * rebuilt from the appsmessages. The entire process takes about 500-700ms. Not validated, with + * the error, when the check could not be made. + * @returns {Promise<{validated: boolean, reindexed: boolean, error?: string}>} */ async function validateAppsInformation() { const response = { validated: false, reindexed: false }; @@ -1062,6 +1299,8 @@ async function validateAppsInformation() { log.info(`validateAppsInformation reindexRequired: ${reindexRequired}`); + if (reindexRequired === null) return response; + if (!reindexRequired) { response.validated = true; return response; @@ -1080,6 +1319,7 @@ async function validateAppsInformation() { response.reindexed = true; } catch (err) { log.error(`Unable to validate apps information. Error: ${err}`); + response.error = err.message; } return response; } @@ -1097,12 +1337,10 @@ async function main(command) { if (command === 'validateInfoCol') { await validateAppsInformation(); } else if (command === 'repairMessagesCol') { - await repairNanInAppsMessagesDb(); + await repairNanInAppsMessagesDb({ force: true }); } - const client = databaseConnection(); - - await client.close(); + await closeDbConnection(); } if (require.main === module) { @@ -1126,7 +1364,13 @@ if (require.main === module) { } module.exports = { + ensureIndex, + ensureIndexes, + dedupeByKey, + maintenanceConnection, + onMaintenanceClient, aggregateInDatabase, + isDatabaseError, appMessagesThatCount, appPaymentFacts, bulkWriteInDatabase, diff --git a/ZelBack/src/services/dockerService.js b/ZelBack/src/services/dockerService.js index 46ebdcd434..77bee91b1c 100644 --- a/ZelBack/src/services/dockerService.js +++ b/ZelBack/src/services/dockerService.js @@ -14,6 +14,7 @@ const { isFluxStorageUrl, storageLinkOf } = require('./utils/fluxStorage'); const log = require('../lib/log'); const cpuBurstHelper = require('./utils/cpuBurstHelper'); const LogFrameDecoder = require('./utils/logFrameDecoder'); +const appSlice = require('./appLifecycle/appSlice'); const globalState = require('./utils/globalState'); @@ -1169,7 +1170,7 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS const appOwner = fullAppSpecs?.owner || null; const burstEligible = appOwner && cpuBurstHelper.isEnterpriseOwner(appOwner) - && await cpuBurstHelper.isCpuBurstSupported(); + && cpuBurstHelper.isCpuBurstEnabled(); const burstLabels = burstEligible ? { 'flux.burst.eligible': 'true', @@ -1239,6 +1240,8 @@ async function appDockerCreate(appSpecifications, appName, isComponent, fullAppS // eslint-disable-next-line no-use-before-define const dockerInfoResp = await dockerInfo(); log.info(dockerInfoResp); + const slice = appSlice.sliceName(appName); + if (slice) options.HostConfig.CgroupParent = slice; const driverStatus = dockerInfoResp.DriverStatus; const backingFs = driverStatus.find((status) => status[0] === 'Backing Filesystem'); // d_type must be true for overlay, docker would not work if not if (backingFs && backingFs[1] === 'xfs') { diff --git a/ZelBack/src/services/explorerService.js b/ZelBack/src/services/explorerService.js index be74343001..3abc877d3c 100644 --- a/ZelBack/src/services/explorerService.js +++ b/ZelBack/src/services/explorerService.js @@ -394,17 +394,29 @@ async function processInsight(blockDataVerbose, database) { // eslint-disable-next-line no-await-in-loop await processSoftFork(tx.txid, blockDataVerbose.height, message); } catch (error) { + // A message that cannot be read is skipped. One that could not be written is not: + // the block is reprocessed, or this node prices apps without that fork for good. + if (dbHelper.isDatabaseError(error)) throw error; log.error('Error processing soft fork message:', error); - // Continue processing other transactions even if soft fork processing fails } } } } } +/** + * Records the app payments of the blocks processed since the last write. A payment that could + * not be recorded rejects, so the block processor reprocesses its block instead of moving its + * scanned height past it: a payment with no record is never resolved, and the app message it + * paid for never reaches this node. + * @param {object[]} transactions + * @param {object} database + * @returns {Promise} + */ async function insertTransactions(transactions, database) { if (transactions.length > 0) { log.info(`Explorer - insertTransactions - Inserting ${transactions.length} transactions to apps hashes collection`); + let unrecorded = null; try { const options = { ordered: false, @@ -424,10 +436,13 @@ async function insertTransactions(transactions, database) { await dbHelper.updateOneInDatabase(database, appsHashesCollection, query, update, options); } catch (errorTx) { log.error(`Explorer - insertTransactions - Inserting ${transaction.hash} - transaction error - ${errorTx}`); + unrecorded = unrecorded ?? errorTx; } } } + // Reprocessing the block collects its payments again. appsTransactions = []; + if (unrecorded) throw unrecorded; } } @@ -436,35 +451,48 @@ async function insertTransactions(transactions, database) { * @param {array} apps array with appstransactions to be processed. * @param {string} database Database. */ +/** + * Stores or requests the messages of a block's app payments. Runs from a timer + * nothing awaits, so it logs its own failure: the hashes it leaves unresolved + * are found again by hash sync. + * @param {array} apps the block's app payments. + * @returns {Promise} + */ +async function requestAppHashes(apps) { + try { + const appsToRemove = []; + let hasUnresolved = false; + // eslint-disable-next-line no-restricted-syntax + for (const app of apps) { + // eslint-disable-next-line no-await-in-loop + const messageReceived = await messageVerifier.checkAndRequestApp(app.hash, app.txid, app.height, app.value, 2); + if (messageReceived) { + appsToRemove.push(app); + } else { + hasUnresolved = true; + } + } + if (hasUnresolved) { + appSyncEvents.emit(SYNC_EVENTS.HASH_UNRESOLVED); + } + const remaining = apps.filter((item) => !appsToRemove.includes(item)); + while (remaining.length > 500) { + messageVerifier.checkAndRequestMultipleApps(remaining.splice(0, 500)); + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(30 * 1000); // delay 30 seconds + } + if (remaining.length > 0) { + messageVerifier.checkAndRequestMultipleApps(remaining); + } + } catch (error) { + log.error(`Explorer - requestAppHashes - ${apps.length} app payments: ${error.message}`); + } +} + async function insertAndRequestAppHashes(apps, database) { if (apps.length > 0) { await insertTransactions(apps, database); - setTimeout(async () => { - const appsToRemove = []; - let hasUnresolved = false; - // eslint-disable-next-line no-restricted-syntax - for (const app of apps) { - // eslint-disable-next-line no-await-in-loop - const messageReceived = await messageVerifier.checkAndRequestApp(app.hash, app.txid, app.height, app.value, 2); - if (messageReceived) { - appsToRemove.push(app); - } else { - hasUnresolved = true; - } - } - if (hasUnresolved) { - appSyncEvents.emit(SYNC_EVENTS.HASH_UNRESOLVED); - } - const remaining = apps.filter((item) => !appsToRemove.includes(item)); - while (remaining.length > 500) { - messageVerifier.checkAndRequestMultipleApps(remaining.splice(0, 500)); - // eslint-disable-next-line no-await-in-loop - await serviceHelper.delay(30 * 1000); // delay 30 seconds - } - if (remaining.length > 0) { - messageVerifier.checkAndRequestMultipleApps(remaining); - } - }, 1); + setTimeout(requestAppHashes, 1, apps); } } /** @@ -550,8 +578,10 @@ async function processStandard(blockDataVerbose, database) { try { await processSoftFork(tx.txid, blockDataVerbose.height, message); } catch (error) { + // A message that cannot be read is skipped. One that could not be written is not: + // the block is reprocessed, or this node prices apps without that fork for good. + if (dbHelper.isDatabaseError(error)) throw error; log.error('Error processing soft fork message:', error); - // Continue processing other transactions even if soft fork processing fails } } } @@ -718,6 +748,124 @@ async function processBlock(blockHeight, isInsightExplorer) { } } +/** + * Removes every payment record but the one a scan keeps for each app message + * hash, and answers how many it removed: a payment this node already holds + * stands (standingAppPayment), so a scan keeps the earliest, and a record + * whose transaction the chain does not hold is no payment and gives way to one + * that is. The stored message of each such hash moves to the record kept before + * the others are removed, and the registry rebuild at boot writes its app's row + * from it. The recovery the unique hash index is built with. + * @param {object} collection the appsHashes collection + * @returns {Promise} + */ +async function keepEarliestPayment(collection) { + const groups = await collection.aggregate([ + { $addFields: { offChain: { $cond: [{ $eq: ['$notOnChain', true] }, 1, 0] } } }, + { $sort: { offChain: 1, height: 1, txIndex: 1, _id: 1 } }, + { + $group: { + _id: '$hash', + rows: { + $push: { + id: '$_id', txid: '$txid', height: '$height', value: '$value', + }, + }, + }, + }, + { $match: { 'rows.1': { $exists: true } } }, + ]).toArray(); + const toRemove = groups.flatMap((group) => group.rows.slice(1).map((row) => row.id)); + if (!toRemove.length) return 0; + // the messages first: until the duplicates are gone, a failure here is found + // again by the next build of the unique index, which runs this again + await messageVerifier.moveStoredMessagesToPayments(groups.map(({ _id: hash, rows: [kept] }) => ({ + hash, txid: kept.txid, height: kept.height, value: kept.value, + }))); + await collection.deleteMany({ _id: { $in: toRemove } }); + return toRemove.length; +} + +/** + * Asserts every index the explorer's collections need. Boot runs it on every + * start, and a scan from zero runs it after emptying them; an index that + * already exists costs a catalog check. Runs on the maintenance client. + * @returns {Promise} + */ +async function prepareCollections() { + const db = dbHelper.databaseConnection(); + const database = dbHelper.onMaintenanceClient(db.db(config.database.daemon.database)); + const databaseUpdates = dbHelper.onMaintenanceClient(db.db(config.database.chainparams.database)); + await dbHelper.ensureIndexes(database.collection(utxoIndexCollection), [ + { + key: { txid: 1, vout: 1 }, name: 'query for getting utxo', unique: true, recover: dbHelper.dedupeByKey, + }, + { + key: { txid: 1, vout: 1, satoshis: 1 }, name: 'query for getting utxo for zelnode tx', unique: true, recover: dbHelper.dedupeByKey, + }, + { key: { address: 1 }, name: 'query for addresses utxo' }, + { key: { scriptPubKey: 1 }, name: 'query for scriptPubKey utxo' }, + ]); + await dbHelper.ensureIndexes(database.collection(coinbaseFusionIndexCollection), [ + { + key: { txid: 1, vout: 1 }, name: 'query for getting coinbase fusion utxo', unique: true, recover: dbHelper.dedupeByKey, + }, + { + key: { txid: 1, vout: 1, satoshis: 1 }, name: 'query for getting coinbase fusion utxo for zelnode tx', unique: true, recover: dbHelper.dedupeByKey, + }, + { key: { address: 1 }, name: 'query for addresses coinbase fusion utxo' }, + { key: { scriptPubKey: 1 }, name: 'query for scriptPubKey coinbase fusion utxo' }, + ]); + await dbHelper.ensureIndexes(database.collection(addressTransactionIndexCollection), [ + { key: { address: 1 }, name: 'query for addresses transactions' }, + { key: { address: 1, count: 1 }, name: 'query for addresses transactions with count' }, + ]); + await dbHelper.ensureIndexes(database.collection(appsHashesCollection), [ + { key: { txid: 1 }, name: 'query for getting txid' }, + { key: { height: 1 }, name: 'query for getting height' }, + { + key: { hash: 1 }, name: 'query for getting app hash', unique: true, recover: keepEarliestPayment, + }, + { key: { message: 1 }, name: 'query for getting app hashes depending if we have message' }, + // a record without a position is indexed under null, so the boot backfill reads only those + { key: { txIndex: 1 }, name: 'query for getting app hashes without a position' }, + ]); + await dbHelper.ensureIndexes(databaseUpdates.collection(chainParamsMessagesCollection), [ + { key: { txid: 1 }, name: 'query for getting txid of some chain parameters update message' }, + { key: { height: 1 }, name: 'query for getting height of some chain parameters update message' }, + { key: { message: 1 }, name: 'query for getting message of some chain parameters update message' }, + { key: { version: 1 }, name: 'query for getting version of some chain parameters update message' }, + ]); +} + +/** + * Empties the explorer's collections and asserts their indexes, for a scan + * from zero. Rejects when either fails, and the scan does not start. + * @returns {Promise} + */ +async function resetExplorerCollections() { + log.info('Preparing daemon collections'); + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.daemon.database); + const databaseUpdates = db.db(config.database.chainparams.database); + const empty = [ + [database, utxoIndexCollection], + [database, addressTransactionIndexCollection], + [database, appsHashesCollection], + [database, coinbaseFusionIndexCollection], + [databaseUpdates, chainParamsMessagesCollection], + ]; + // eslint-disable-next-line no-restricted-syntax + for (const [owner, collection] of empty) { + // eslint-disable-next-line no-await-in-loop + await dbHelper.dropCollection(owner, collection).catch((error) => { + if (error.message !== 'ns not found') throw error; + }); + } + await prepareCollections(); + log.info('Preparation done'); +} + /** * To restore database to specified block height. * @param {number} height Block height. @@ -727,8 +875,9 @@ async function restoreDatabaseToBlockheightState(height) { if (!height) { throw new Error('No blockheight for restoring provided'); } + // Whole-collection deletes and updates: they run on the maintenance client. const dbopen = dbHelper.databaseConnection(); - const database = dbopen.db(config.database.daemon.database); + const database = dbHelper.onMaintenanceClient(dbopen.db(config.database.daemon.database)); const query = { height: { $gt: height } }; const queryForAddresses = {}; // we need to remove those transactions in transactions field that have height greater than height @@ -747,7 +896,7 @@ async function restoreDatabaseToBlockheightState(height) { // restore appsHashes collection await dbHelper.removeDocumentsFromCollection(database, appsHashesCollection, query); log.info('Rescanning Blockchain Parameters!'); - const databaseUpdates = dbopen.db(config.database.chainparams.database); + const databaseUpdates = dbHelper.onMaintenanceClient(dbopen.db(config.database.chainparams.database)); await dbHelper.removeDocumentsFromCollection(databaseUpdates, chainParamsMessagesCollection, query); log.info('Rescan completed'); return true; @@ -1142,6 +1291,9 @@ async function recoverAndRestart(restoreDatabase, deepRestore) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.daemon.database); let scannedBlockHeight = await getScannedBlockHeightFromDb(database); + // A scan from zero starts on empty collections holding every index, however + // the attempt before this one ended. + if (scannedBlockHeight === 0) await resetExplorerCollections(); if (scannedBlockHeight !== 0 && restoreDatabase) { const deepRestoreBlocks = config.fluxapps.explorerDeepRestoreBlocks ?? 100; @@ -1204,75 +1356,8 @@ async function initiateBlockProcessor(restoreDatabase, deepRestore) { } const daemonHeight = daemonBlockCount.data; - if (scannedBlockHeight === 0) { - log.info('Preparing daemon collections'); - const result = await dbHelper.dropCollection(database, utxoIndexCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const resultB = await dbHelper.dropCollection(database, addressTransactionIndexCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const resultD = await dbHelper.dropCollection(database, appsHashesCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const resultFusion = await dbHelper.dropCollection(database, coinbaseFusionIndexCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - const databaseUpdates = db.db(config.database.chainparams.database); - const resultChainParams = await dbHelper.dropCollection(databaseUpdates, chainParamsMessagesCollection).catch((error) => { - if (error.message !== 'ns not found') throw error; - }); - log.info(result, resultB, resultD, resultFusion, resultChainParams); - - await database.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting utxo', unique: true }); - await database.collection(utxoIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting utxo for zelnode tx', unique: true }); - await database.collection(utxoIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses utxo' }); - await database.collection(utxoIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey utxo' }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1 }, { name: 'query for getting coinbase fusion utxo', unique: true }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ txid: 1, vout: 1, satoshis: 1 }, { name: 'query for getting coinbase fusion utxo for zelnode tx', unique: true }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses coinbase fusion utxo' }); - await database.collection(coinbaseFusionIndexCollection).createIndex({ scriptPubKey: 1 }, { name: 'query for scriptPubKey coinbase fusion utxo' }); - await database.collection(addressTransactionIndexCollection).createIndex({ address: 1 }, { name: 'query for addresses transactions' }); - await database.collection(addressTransactionIndexCollection).createIndex({ address: 1, count: 1 }, { name: 'query for addresses transactions with count' }); - await database.collection(appsHashesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid' }); - await database.collection(appsHashesCollection).createIndex({ height: 1 }, { name: 'query for getting height' }); - await database.collection(appsHashesCollection).createIndex({ hash: 1 }, { name: 'query for getting app hash', unique: true }).catch((error) => { - log.error('Expected throw on index creation as of new uniquness. Do not remove this check until all nodes have rebuild apps data'); - log.error(error); - }); - await database.collection(appsHashesCollection).createIndex({ message: 1 }, { name: 'query for getting app hashes depending if we have message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ txid: 1 }, { name: 'query for getting txid of some chain parameters update message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ height: 1 }, { name: 'query for getting height of some chain parameters update message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ message: 1 }, { name: 'query for getting message of some chain parameters update message' }); - await databaseUpdates.collection(chainParamsMessagesCollection).createIndex({ version: 1 }, { name: 'query for getting version of some chain parameters update message' }); - - const databaseGlobal = db.db(config.database.appsglobal.database); - log.info('Preparing apps collections'); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ hash: 1 }, { name: 'query for getting zelapp message based on hash', unique: true }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ txid: 1 }, { name: 'query for getting zelapp message based on txid' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ height: 1 }, { name: 'query for getting zelapp message based on height' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.name': 1 }, { name: 'query for getting app message based on zelapp specs name' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.owner': 1 }, { name: 'query for getting app message based on zelapp specs owner' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.repotag': 1 }, { name: 'query for getting app message based on image' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.version': 1 }, { name: 'query for getting app message based on version' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsMessages).createIndex({ 'appSpecifications.nodes': 1 }, { name: 'query for getting app message based on nodes' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ name: 1 }, { name: 'query for getting zelapp based on zelapp specs name' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ owner: 1 }, { name: 'query for getting zelapp based on zelapp specs owner' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ repotag: 1 }, { name: 'query for getting zelapp based on image' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ height: 1 }, { name: 'query for getting zelapp based on last height update' }); - await databaseGlobal.collection(config.database.appsglobal.collections.appsInformation).createIndex({ hash: 1 }, { name: 'query for getting zelapp based on last hash' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp location based on zelapp specs name' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ hash: 1 }, { name: 'query for getting zelapp location based on zelapp hash' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ ip: 1 }, { name: 'query for getting zelapp location based on ip' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting app based on ip and name' }); - await database.collection(config.database.appsglobal.collections.appsLocations).createIndex({ name: 1, ip: 1, broadcastedAt: 1 }, { name: 'query for getting app to ensure we possess a message' }); - await database.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1 }, { name: 'query for getting zelapp install location based on zelapp specs name' }); - await database.collection(config.database.appsglobal.collections.appsInstallingLocations).createIndex({ name: 1, ip: 1 }, { name: 'query for getting flux app install location based on specs name and node ip' }); - await database.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1 }, { name: 'query for getting flux app install errors location based on specs name' }); - await database.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash' }); - await database.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).createIndex({ name: 1, hash: 1, ip: 1 }, { name: 'query for getting flux app install errors location based on specs name and hash and node ip' }); - log.info('Preparation done'); - } + // A scan from zero starts on empty collections holding every index. + if (scannedBlockHeight === 0) await resetExplorerCollections(); if (daemonHeight > scannedBlockHeight) { if (scannedBlockHeight !== 0 && restoreDatabase) { @@ -1676,6 +1761,8 @@ async function checkBlockProcessingStopped(i, callback) { if (j < 12) { checkBlockProcessingStopped(j, callback); } else { + // Not stopped: the block in flight keeps chaining to the next one. + blockProccessingCanContinue = true; const errMessage = messageHelper.createErrorMessage('Unknown error occured. Try again later.'); callback(errMessage); } @@ -1764,25 +1851,23 @@ async function reindexExplorer(req, res) { res.json(errMessage); } else { operationBlocked = true; - const dbopen = dbHelper.databaseConnection(); - const database = dbopen.db(config.database.daemon.database); - const resultOfDropping = await dbHelper.dropCollection(database, scannedHeightCollection).catch((error) => { - if (error.message !== 'ns not found') { - operationBlocked = false; - log.error(error); - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - res.json(errMessage); - } - }); - operationBlocked = false; - if (resultOfDropping === true || resultOfDropping === undefined) { - initiateBlockProcessor(true, false); - const message = messageHelper.createSuccessMessage('Explorer database reindex initiated'); - res.json(message); - } else { - const errMessage = messageHelper.createErrorMessage(resultOfDropping, 'Collection dropping error'); - res.json(errMessage); + try { + const dbopen = dbHelper.databaseConnection(); + const database = dbopen.db(config.database.daemon.database); + await dbHelper.dropCollection(database, scannedHeightCollection).catch((error) => { + // A collection that is already gone is the reindex's starting point. + if (error.message !== 'ns not found') throw error; + }); + } catch (error) { + operationBlocked = false; + log.error(error); + res.json(messageHelper.createErrorMessage(error.message, error.name, error.code)); + return; } + operationBlocked = false; + initiateBlockProcessor(true, false); + const message = messageHelper.createSuccessMessage('Explorer database reindex initiated'); + res.json(message); } }); } else { @@ -1813,6 +1898,7 @@ async function rescanExplorer(req, res) { if (!blockheight) { const errMessage = messageHelper.createErrorMessage('No blockheight provided'); res.json(errMessage); + return; } blockheight = serviceHelper.ensureNumber(blockheight); const dbopen = dbHelper.databaseConnection(); @@ -1839,17 +1925,23 @@ async function rescanExplorer(req, res) { checkBlockProcessingStopped(i, async (response) => { if (response.status === 'error') { res.json(response); + } else if (operationBlocked) { + res.json(messageHelper.createErrorMessage('Operation blocked')); } else { - if (operationBlocked) { - throw new Error('Operation blocked'); - } operationBlocked = true; const update = { $set: { generalScannedHeight: blockheight } }; const options = { upsert: true, }; - // update scanned Height in scannedBlockHeightCollection - await dbHelper.updateOneInDatabase(database, scannedHeightCollection, query, update, options); + try { + // update scanned Height in scannedBlockHeightCollection + await dbHelper.updateOneInDatabase(database, scannedHeightCollection, query, update, options); + } catch (error) { + operationBlocked = false; + log.error(error); + res.json(messageHelper.createErrorMessage(error.message, error.name, error.code)); + return; + } operationBlocked = false; initiateBlockProcessor(true, false); const message = messageHelper.createSuccessMessage(`Explorer rescan from blockheight ${blockheight} initiated`); @@ -1938,6 +2030,11 @@ function setBlockProccessingCanContinue(value) { blockProccessingCanContinue = value; } +// testing purposes +function getBlockProccessingCanContinue() { + return blockProccessingCanContinue; +} + // testing purposes function setIsInInitiationOfBP(value) { isInInitiationOfBP = value; @@ -1949,6 +2046,9 @@ function setZelAppSpecsMigrationDone(value) { } module.exports = { + prepareCollections, + resetExplorerCollections, + keepEarliestPayment, initiateBlockProcessor, processBlock, reindexExplorer, @@ -1980,6 +2080,8 @@ module.exports = { processTransaction, processStandard, setBlockProccessingCanContinue, + getBlockProccessingCanContinue, + insertTransactions, setIsInInitiationOfBP, setZelAppSpecsMigrationDone, restoreDatabaseToBlockheightState, diff --git a/ZelBack/src/services/fluxCommunication.js b/ZelBack/src/services/fluxCommunication.js index 9c6bd5b0c1..d31aa5a960 100644 --- a/ZelBack/src/services/fluxCommunication.js +++ b/ZelBack/src/services/fluxCommunication.js @@ -4,7 +4,9 @@ const hash = require('object-hash'); const WebSocket = require('ws'); const log = require('../lib/log'); const serviceHelper = require('./serviceHelper'); +const dbHelper = require('./dbHelper'); const messageStore = require('./appMessaging/messageStore'); +const { appRunningEntries } = require('./utils/appRunningEntries'); const verificationHelper = require('./verificationHelper'); const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); const fluxCommunicationMessagesSender = require('./fluxCommunicationMessagesSender'); @@ -80,6 +82,14 @@ async function handleAppMessages(message, fromIP, port) { } } +/** + * The broadcasts of a sync response that verify. Rejects when any of them + * could not be verified because the database did not answer: the response is + * then not all recorded, and its peer has not answered. + * @param {object[]} broadcasts + * @param {string} label + * @returns {Promise} + */ async function batchVerifyBroadcasts(broadcasts, label) { if (broadcasts.length === 0) return []; @@ -87,6 +97,8 @@ async function batchVerifyBroadcasts(broadcasts, label) { const verdicts = await fluxCommunicationUtils.verifyBroadcasts(broadcasts, { verifySignatures: (items) => verifyPool.verify(items), }); + const unreadable = verdicts.filter(({ result }) => result === VerifyResult.UNREADABLE).length; + if (unreadable > 0) throw new Error(`${label} - ${unreadable} broadcasts could not be verified, the database did not answer`); const verified = broadcasts.filter((broadcast, i) => verdicts[i].result === VerifyResult.OK); const signatureFailures = verdicts.filter(({ result }) => result === VerifyResult.BAD_SIGNATURE).length; const nodeCheckFailures = broadcasts.length - verified.length - signatureFailures; @@ -118,14 +130,28 @@ async function handleTempSyncResponse(message, peerSocket) { if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleTempSyncResponse - Received ${messages.length} temp messages from ${peerKey} (done: ${!!done})`); let stored = 0; + let writeFailed = false; for (const msg of messages) { try { const result = await messageStore.storeAppTemporaryMessage(msg, { furtherVerification: true }); if (result === true || result === false) stored += 1; } catch (err) { log.error(`Temp sync message failed: ${err.message}`); + // A message this node's database could not record leaves the answer + // incomplete; one that fails its own checks is the peer's, and is skipped. + if (dbHelper.isDatabaseError(err)) { + writeFailed = true; + break; + } } } + if (writeFailed) { + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleTempSyncResponse - a write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; + } log.info(`handleTempSyncResponse - Processed ${stored} of ${messages.length} messages`); // COUNTED LIKE THE REST. A peer is credited once it has delivered every // stream it was asked for, so the record that admits its responses stays @@ -140,6 +166,20 @@ async function handleTempSyncResponse(message, peerSocket) { } } +/** + * Ends the request of a peer whose sync response threw part way: what it sent is + * not all recorded, so another peer is asked, and the rest of its answer is not + * read. A peer whose response this node no longer wants has no request to end. + * @param {{key: string, connectionId?: number}} peerSocket + * @param {string} handler Names the handler in the log. + * @returns {void} + */ +function endUnrecordedRequest(peerSocket, handler) { + if (!peerManager.isSyncResponseWanted(peerSocket)) return; + log.warn(`${handler} - the response was not all recorded, ending ${peerSocket.key}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerSocket.key); +} + async function handleAppRunningSyncResponse(message, peerSocket) { try { if (!message.data || message.data.type !== 'fluxapprunningsync') return; @@ -223,9 +263,14 @@ async function handleAppRunningSyncResponse(message, peerSocket) { // already-expired rows - but that is a bound to be read out of another // file, and two builders of one input should not need reconciling. if (data.broadcastedAt + RUNNING_EXPIRY_MS < Date.now()) continue; + // Pruned against the apps the broadcast names that can be stored: one + // naming only entries this node refuses says nothing about what its + // sender runs, and is never read as one naming no apps at all. + const { stored } = appRunningEntries(data.apps); + if (!stored.length) continue; const seen = newestByIp.get(data.ip); if (!seen || data.broadcastedAt > seen.broadcastedAt) { - newestByIp.set(data.ip, { names: data.apps.map((a) => a.name), broadcastedAt: data.broadcastedAt }); + newestByIp.set(data.ip, { names: stored.map((a) => a.name), broadcastedAt: data.broadcastedAt }); } } @@ -238,10 +283,13 @@ async function handleAppRunningSyncResponse(message, peerSocket) { }); if (locationWriteFailed) { - log.warn('handleAppRunningSyncResponse - skipping location pruning, a location write failed'); - } else { - await messageStore.pruneAppRunningLocations(newestByIp); + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleAppRunningSyncResponse - a location write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; } + await messageStore.pruneAppRunningLocations(newestByIp); // Applied after every slice, once every apprunning row in the response is // stored. Each event removes, shortens or carries only the rows broadcast @@ -267,6 +315,7 @@ async function handleAppRunningSyncResponse(message, peerSocket) { } } catch (error) { log.error(error); + endUnrecordedRequest(peerSocket, 'handleAppRunningSyncResponse'); } } @@ -284,19 +333,29 @@ async function handleAppInstallingSyncResponse(message, peerSocket) { } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppInstallingSyncResponse - Received ${messages.length} broadcasts from ${peerKey} (done: ${!!done})`); + let writeFailed = false; await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { const verified = await batchVerifyBroadcasts(slice, 'handleAppInstallingSyncResponse'); if (verified.length === 0) return; - const { stored } = await messageStore.storeBatchAppInstallingMessages(verified); - log.info(`handleAppInstallingSyncResponse - Stored ${stored} of ${verified.length} verified broadcasts`); - fluxEventBus.publish('sync:chunkVerified', { syncType: 'appinstalling', peer: peerKey, verified: verified.length, stored }); + const stored = await messageStore.storeBatchAppInstallingMessages(verified); + if (stored.writeFailed) writeFailed = true; + log.info(`handleAppInstallingSyncResponse - Stored ${stored.stored} of ${verified.length} verified broadcasts`); + fluxEventBus.publish('sync:chunkVerified', { syncType: 'appinstalling', peer: peerKey, verified: verified.length, stored: stored.stored }); }); + if (writeFailed) { + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleAppInstallingSyncResponse - a write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; + } if (done) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'appinstalling', peerKey); log.info('handleAppInstallingSyncResponse - Sync complete'); } } catch (error) { log.error(error); + endUnrecordedRequest(peerSocket, 'handleAppInstallingSyncResponse'); } } @@ -314,19 +373,29 @@ async function handleAppInstallingErrorsSyncResponse(message, peerSocket) { } if (!Array.isArray(messages) || messages.length > 2500) return; log.info(`handleAppInstallingErrorsSyncResponse - Received ${messages.length} broadcasts from ${peerKey} (done: ${!!done})`); + let writeFailed = false; await serviceHelper.processInSlices(messages, SYNC_EVENTS_PER_SLICE, async (slice) => { const verified = await batchVerifyBroadcasts(slice, 'handleAppInstallingErrorsSyncResponse'); if (verified.length === 0) return; - const { stored } = await messageStore.storeBatchAppInstallingErrorMessages(verified); - log.info(`handleAppInstallingErrorsSyncResponse - Stored ${stored} of ${verified.length} verified broadcasts`); - fluxEventBus.publish('sync:chunkVerified', { syncType: 'apperrors', peer: peerKey, verified: verified.length, stored }); + const stored = await messageStore.storeBatchAppInstallingErrorMessages(verified); + if (stored.writeFailed) writeFailed = true; + log.info(`handleAppInstallingErrorsSyncResponse - Stored ${stored.stored} of ${verified.length} verified broadcasts`); + fluxEventBus.publish('sync:chunkVerified', { syncType: 'apperrors', peer: peerKey, verified: verified.length, stored: stored.stored }); }); + if (writeFailed) { + // What this peer sent is not all recorded, so its request ends here and + // another peer is asked; the rest of its answer is not read. + log.warn(`handleAppInstallingErrorsSyncResponse - a write failed, ending ${peerKey}'s request`); + appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peerKey); + return; + } if (done) { appSyncEvents.emit(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, 'apperrors', peerKey); log.info('handleAppInstallingErrorsSyncResponse - Sync complete'); } } catch (error) { log.error(error); + endUnrecordedRequest(peerSocket, 'handleAppInstallingErrorsSyncResponse'); } } @@ -397,14 +466,29 @@ async function handleAppRunningMessage(message, fromIP, port) { try { await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: message }); const result = await messageStore.storeAppRunningMessage(message.data); - if (result.stored) { - fluxEventBus.publish('network:apprunning', { ip: message.data.ip, apps: message.data.apps || [{ name: message.data.name }] }); + if (result instanceof Error) { + log.warn(`handleAppRunningMessage - not stored, from ${fromIP}:${port} for ${message.data?.ip}: ${result.message}`); + return; } const currentTimeStamp = Date.now(); const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(message, currentTimeStamp, 240000); - if (result.rebroadcast && timestampOK) { + const relayed = Boolean(result.rebroadcast && timestampOK); + if (relayed) { announceToPeers(message, `${fromIP}:${port}`); } + // Everything this node did with the message: the entries it wrote, the names + // it refused, whether it cleared the sender's rows, and whether it passed it on. + // A message that did none of these, such as a copy of one already stored, + // publishes nothing. + if (result.written.length || result.refused.length || result.cleared || relayed) { + fluxEventBus.publish('network:apprunning', { + ip: message.data.ip, + apps: result.written, + refused: result.refused, + cleared: result.cleared, + relayed, + }); + } } catch (error) { log.error(error); } @@ -428,7 +512,9 @@ async function handleAppInstallingMessage(message, fromIP, port) { withdrawn: message.data.withdrawn === true, }); } - messageStore.storeSignedAppInstallingBroadcast(message); + // The signed claim is what this node serves to peers syncing from it: one that is + // not recorded is not relayed as though it were. + await messageStore.storeSignedAppInstallingBroadcast(message); const currentTimeStamp = Date.now(); const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(message, currentTimeStamp); if (rebroadcastToPeers === true && timestampOK) { @@ -451,7 +537,7 @@ async function handleAppInstallingErrorMessage(message, fromIP, port) { if (rebroadcastToPeers === true) { fluxEventBus.publish('network:appinstallingerror', { ip: message.data.ip, name: message.data.name }); } - messageStore.storeSignedAppInstallingErrorBroadcast(message); + await messageStore.storeSignedAppInstallingErrorBroadcast(message); const currentTimeStamp = Date.now(); const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(message, currentTimeStamp); if (rebroadcastToPeers === true && timestampOK) { @@ -555,7 +641,6 @@ function dropExpiredOwnMoves() { * key signed is the previous holder's. */ async function loadRecordedOwnMoves() { - recordedLoaded = true; try { const localSocketAddr = await fluxNetworkHelper.getLocalSocketAddress(); const ownKey = await fluxNetworkHelper.getFluxNodePublicKey(); @@ -565,6 +650,8 @@ async function loadRecordedOwnMoves() { for (const [oldIp, { to, at, signer }] of recorded) { if (signer === ownKey) addPendingOwnMove(oldIp, to, at); } + // Only a completed read counts: until then each connecting peer asks again. + recordedLoaded = true; } catch (error) { log.warn(`loadRecordedOwnMoves - this node's recorded moves cannot be read: ${error.message}`); } @@ -591,7 +678,7 @@ async function sendOwnMovesTo(peerKey) { log.error(`sendOwnMovesTo - the move from ${move.oldIp} to ${move.newIp} could not be sent to ${peerKey}: ${error.message}`); } } - if (!pendingOwnMoves.length) stopSendingOwnMoves(); + stopSendingOwnMovesWhenDone(); } function startSendingOwnMoves() { @@ -606,6 +693,14 @@ function stopSendingOwnMoves() { peerManager.off('peerConnected', sendOwnMovesTo); } +/** + * Stop listening for peers once nothing is left to send: every pending move is + * gone and the recorded ones have been read. + */ +function stopSendingOwnMovesWhenDone() { + if (recordedLoaded && !pendingOwnMoves.length) stopSendingOwnMoves(); +} + /** * A peer announced, or relayed, a message this node holds the hash of. When it * is one of this node's own moves, the move has gone further than the peers @@ -619,7 +714,7 @@ function noteOwnMoveHeardBack(messageHash) { if (index === -1) return false; const [move] = pendingOwnMoves.splice(index, 1); log.info(`Address move from ${move.oldIp} to ${move.newIp} heard back from the network - no longer sent`); - if (!pendingOwnMoves.length) stopSendingOwnMoves(); + stopSendingOwnMovesWhenDone(); return true; } @@ -783,6 +878,10 @@ async function handleNodeSigtermMessage(message, fromIP, port) { // byte budget is ever wanted it belongs in lruRateLimit, covering every type, not in a // second counter here. +// How long a broadcast this node could not verify, because its database did +// not answer, waits before it is dispatched again. +const BROADCAST_VERIFY_RETRY_MS = 30 * 1000; + async function dispatchFluxMessage(msgObj, peerSocket) { const codes = peerSocket.closeCodes; const { @@ -872,7 +971,20 @@ async function dispatchFluxMessage(msgObj, peerSocket) { const { VerifyResult } = fluxCommunicationUtils; const verifyResult = await fluxCommunicationUtils.verifyFluxBroadcast(msgObj, undefined, currentTimeStamp); - if (verifyResult === VerifyResult.OK) { + if (verifyResult === VerifyResult.UNREADABLE) { + // The slot stays claimed while the broadcast waits, so its copies are not + // verified meanwhile; it is dispatched afresh while it is within its window. + if (fluxCommunicationUtils.verifyTimestampInFluxBroadcast(msgObj, currentTimeStamp + BROADCAST_VERIFY_RETRY_MS)) { + log.warn(`Deferring a ${msgObj.data?.type} from ${peerSocket.direction} peer ${peerSocket.key}: the database did not answer`); + setTimeout(() => { + releaseSlot(); + dispatchFluxMessage(msgObj, peerSocket).catch((error) => log.error(error)); + }, BROADCAST_VERIFY_RETRY_MS); + } else { + releaseSlot(); + log.warn(`Dropping a ${msgObj.data?.type} from ${peerSocket.direction} peer ${peerSocket.key}: the database did not answer within its window`); + } + } else if (verifyResult === VerifyResult.OK) { const timestampOK = fluxCommunicationUtils.verifyTimestampInFluxBroadcast(msgObj, currentTimeStamp); if (timestampOK === true) { const handler = handlerFor(msgObj.data.type); @@ -2105,6 +2217,7 @@ module.exports = { logSocketsEvery, handleAppRunningMessage, handleAppInstallingMessage, + handleAppInstallingErrorMessage, handleTempSyncResponse, handleAppRunningSyncResponse, handleAppInstallingSyncResponse, diff --git a/ZelBack/src/services/fluxCommunicationMessagesSender.js b/ZelBack/src/services/fluxCommunicationMessagesSender.js index 6f74309e2c..85543c0493 100644 --- a/ZelBack/src/services/fluxCommunicationMessagesSender.js +++ b/ZelBack/src/services/fluxCommunicationMessagesSender.js @@ -454,7 +454,7 @@ async function respondWithTempMessages(peer, sinceTimestamp = 0) { .find(query, { projection: { _id: 0, receivedAt: 0, expireAt: 0 } }) .sort({ receivedAt: 1 }); - const batchSize = 2000; + const batchSize = config.fluxapps.syncAnswerBatchSize; let batch = []; let total = 0; for await (const msg of cursor) { @@ -515,7 +515,7 @@ async function streamBatchedSync(peer, { sinceTimestamp, collectionName, validit .find(finalQuery, { projection: finalProjection }) .sort({ broadcastedAt: 1 }); - const batchSize = 2000; + const batchSize = config.fluxapps.syncAnswerBatchSize; let batch = []; let total = 0; for await (const doc of cursor) { diff --git a/ZelBack/src/services/fluxCommunicationUtils.js b/ZelBack/src/services/fluxCommunicationUtils.js index b952664555..1fb623e908 100644 --- a/ZelBack/src/services/fluxCommunicationUtils.js +++ b/ZelBack/src/services/fluxCommunicationUtils.js @@ -106,6 +106,7 @@ const VerifyResult = Object.freeze({ NODE_NOT_FOUND: 'nodeNotFound', BAD_SIGNATURE: 'badSignature', PUBKEY_MISMATCH: 'pubkeyMismatch', + UNREADABLE: 'unreadable', }); /** @@ -136,28 +137,23 @@ function signerAddresses(payload, target) { * only when an apprunning came after its eviction and after any move away from * it, so the newest apprunning is the one that wrote them. * @param {string} address - * @returns {Promise} Null when no rows stand there, no apprunning - * is stored for it, or either cannot be read. + * @returns {Promise} Null when no rows stand there or no apprunning + * is stored for it. Rejects when either cannot be read. */ async function lastSignerAt(address) { - try { - const database = dbHelper.databaseConnection().db(config.database.appsglobal.database); - const row = await database.collection(config.database.appsglobal.collections.appsLocations) - .findOne({ ip: address }, { projection: { _id: 0, ip: 1 } }); - if (!row) return null; - const [event] = await database.collection(config.database.appsglobal.collections.appStateEvents) - .find( - { ip: address, type: 'apprunning', expireAt: { $gt: new Date() } }, - { projection: { _id: 0, 'envelope.pubKey': 1 } }, - ) - .sort({ broadcastedAt: -1 }) - .limit(1) - .toArray(); - return event?.envelope?.pubKey ?? null; - } catch (error) { - log.error(`lastSignerAt ${address}: ${error.message}`); - return null; - } + const database = dbHelper.databaseConnection().db(config.database.appsglobal.database); + const row = await database.collection(config.database.appsglobal.collections.appsLocations) + .findOne({ ip: address }, { projection: { _id: 0, ip: 1 } }); + if (!row) return null; + const [event] = await database.collection(config.database.appsglobal.collections.appStateEvents) + .find( + { ip: address, type: 'apprunning', expireAt: { $gt: new Date() } }, + { projection: { _id: 0, 'envelope.pubKey': 1 } }, + ) + .sort({ broadcastedAt: -1 }) + .limit(1) + .toArray(); + return event?.envelope?.pubKey ?? null; } /** @@ -206,6 +202,7 @@ async function listedSignerVerdict(addresses, pubKey) { * standing pubKey OK * standing another key PUBKEY_MISMATCH * none - NODE_NOT_FOUND + * unreadable - UNREADABLE * * With no rows at the address a move left, the move has nothing to carry, and * the chain already lists its node where it is. @@ -214,7 +211,12 @@ async function listedSignerVerdict(addresses, pubKey) { * @returns {Promise<{result: string, reason?: string}>} */ async function lastSignerVerdict(address, pubKey) { - const lastSigner = await lastSignerAt(address); + let lastSigner; + try { + lastSigner = await lastSignerAt(address); + } catch (error) { + return { result: VerifyResult.UNREADABLE, reason: `The location rows at ${address}, which a move signed by ${pubKey} left, cannot be read: ${error.message}` }; + } if (lastSigner === null) { return { result: VerifyResult.NODE_NOT_FOUND, reason: `No location rows stand at ${address}, which a move signed by ${pubKey} left` }; } @@ -312,8 +314,10 @@ function verifySignaturesInline(items) { * signature, then anything that reads the database. A broadcast whose signature * fails never costs a database read. * - * NODE_NOT_FOUND is the one refusal that does not count against the peer that - * relayed the broadcast: the list is cached, and can lag the chain. + * NODE_NOT_FOUND and UNREADABLE are the refusals that do not count against the + * peer that relayed the broadcast: the list is cached, and can lag the chain, + * and UNREADABLE is this node's database not answering, which says nothing of + * the broadcast. * @param {FluxNetworkMessage[]} broadcasts * @param {object} [options] * @param {function(object[]): (boolean[]|Promise)} [options.verifySignatures] diff --git a/ZelBack/src/services/fluxNetworkHelper.js b/ZelBack/src/services/fluxNetworkHelper.js index 19635393c1..376dc15217 100644 --- a/ZelBack/src/services/fluxNetworkHelper.js +++ b/ZelBack/src/services/fluxNetworkHelper.js @@ -28,6 +28,7 @@ const fluxEventBus = require('./utils/fluxEventBus'); const { NON_PUBLIC_IPV4 } = require('./utils/nonPublicNetworks'); const ufw = require('./utils/ufw'); const ufwHelper = require('./utils/ufwHelper'); +const { UDP_PROBE_MAGIC, UDP_PROBE_BYTES } = require('./utils/fluxUdpTestServer'); const { normalizeSocketAddress, extractIp, extractPort, socketAddressesMatch, parseSocketAddress, ipsMatch, } = require('./utils/socketAddressUtils'); @@ -69,13 +70,18 @@ function setInstalledAppsListedAt(callback) { * Whether this node ran apps at `socketAddress`: an app installed here is one * the network lists there. A configuration names an address, and a machine * started on a copy of another's names that machine's; only apps left there - * show this node held it. + * show this node held it. `null` when that cannot be read. * @param {string} socketAddress - * @returns {Promise} + * @returns {Promise} */ async function ranAppsAt(socketAddress) { if (!installedAppsListedAt) return false; - return (await installedAppsListedAt(socketAddress)).length > 0; + try { + return (await installedAppsListedAt(socketAddress)).length > 0; + } catch (error) { + log.warn(`own address - whether this node ran apps at ${socketAddress} cannot be read: ${error.message}`); + return null; + } } let dosState = 0; // we can start at bigger number later @@ -92,6 +98,9 @@ const StickyDosOwner = Object.freeze({ PEER_SET_STABILITY: 'peerSetStability', NODEJS_FLOOR: 'nodejsFloor', DOCKER_FLOOR: 'dockerFloor', + MONGODB_FLOOR: 'mongodbFloor', + CGROUP_FLOOR: 'cgroupFloor', + HOST_STALL: 'hostStall', }); // Declares the node unfit for the apps it already runs, and is not moved by the @@ -116,6 +125,8 @@ const STICKY_DOS_STATE = 100; // a new one. const PlacementHoldOwner = Object.freeze({ RESIDENTIAL_DOS: 'residentialDos', + DATABASE_SUSPECT: 'databaseSuspect', + HOST_STALL: 'hostStall', }); // Stops the node taking on NEW apps without declaring it unfit for the ones it @@ -137,6 +148,7 @@ let storedFluxBenchAllowed = null; let tooManyIpChangesAt = null; let maxNumberOfIpChanges = 0; const { lruRateLimit } = require('./utils/rateLimit'); +const { AsyncGate } = require('./utils/asyncGate'); const { Privilege, authOf } = require('./utils/privileges'); // This node's socket address (ip:port) from benchmark @@ -158,8 +170,11 @@ const EGRESS_TARGET = '203.0.113.1'; // localSocketAddress but gives the node no new address, so this keeps it. let lastKnownSocketAddress = null; // Emits 'change' with the new address when the node learns one that differs -// from lastKnownSocketAddress. +// from lastKnownSocketAddress, and 'known' with it when the node has an address +// after having none. const localSocketAddressEvents = new EventEmitter(); +// Open while localSocketAddress holds an address. +const localSocketAddressGate = new AsyncGate(); /** * The device an os.networkInterfaces() entry is bound on. A labelled address @@ -422,6 +437,98 @@ async function isPortOpen(ip, port, options = {}) { return connected; } +/** + * When each datagram of a UDP probe is sent, in ms from the first. UDP drops + * packets and loss comes in bursts, so the sends are spread rather than sent + * together. + */ +const UDP_PROBE_SCHEDULE_MS = [0, 250, 750, 1750]; + +/** How long a UDP probe waits for a reply after its first datagram. */ +const UDP_PROBE_DEADLINE_MS = 3_000; + +/** + * What a UDP port answered, capped, for the requester to judge - the UDP twin + * of portAnswered. + * + * UDP has no handshake, so silence is all a closed port and a lost datagram + * ever say. The requester's test server answers a probe with its secret, and + * only that tells the requester the port is reachable; this hands back what + * came, and the requester compares. + * + * The reply is taken from whatever source port it arrives from: a NAT in front + * of the requester may rewrite it. + * + * @param {string} ip - the requester's address + * @param {number} port - the port to probe + * @param {{schedule?: number[], deadline?: number}} [options] + * @returns {Promise} what it answered; null when probes + * went out and nothing came back; undefined when no probe could be sent, which + * says nothing about the port + */ +async function portAnsweredUdp(ip, port, options = {}) { + const schedule = options.schedule ?? UDP_PROBE_SCHEDULE_MS; + const deadline = options.deadline ?? UDP_PROBE_DEADLINE_MS; + + const probe = Buffer.alloc(UDP_PROBE_BYTES); + probe.write(UDP_PROBE_MAGIC, 'latin1'); + + const socket = dgram.createSocket('udp4'); + + return new Promise((resolve) => { + const timers = []; + let settled = false; + let sent = 0; + let failed = 0; + + const done = (answer) => { + if (settled) return; + settled = true; + timers.forEach(clearTimeout); + socket.close(); + resolve(answer); + }; + + const afterSend = (error) => { + if (!error) { + sent += 1; + return; + } + failed += 1; + if (failed === schedule.length) done(undefined); + }; + + socket.on('message', (message, rinfo) => { + if (rinfo.address.replace(/^::ffff:/i, '') !== ip) return; + done(message.subarray(0, MAX_ECHO_BYTES).toString('utf8')); + }); + socket.on('error', () => done(undefined)); + + schedule.forEach((at) => { + timers.push(setTimeout(() => socket.send(probe, port, ip, afterSend), at)); + }); + timers.push(setTimeout(() => done(sent ? null : undefined), deadline)); + }); +} + +/** + * Runs `task` once per item, at most `limit` at a time. + * @param {Array} items + * @param {number} limit + * @param {function(*): Promise} task + * @returns {Promise} + */ +async function forEachLimited(items, limit, task) { + const queue = [...items]; + async function worker() { + while (queue.length) { + // eslint-disable-next-line no-await-in-loop + await task(queue.shift()); + } + } + await Promise.all(Array.from({ length: Math.min(limit, queue.length) }, worker)); +} + /** * To perform a basic check of current FluxOS version. * @param {string} ip IP address. @@ -557,6 +664,14 @@ const MAX_TESTABLE_PORTS = 50; // how long one request can keep this node poking, and nothing tighter: the // honest list really can be that long. const NODE_SERVICE_PORTS_KEPT_ALIVE = 4; + +/** + * How many of one request's ports a peer probes at once, where it probes them + * in parallel: UDP always, TCP when asked for every port's answer. A port that + * never answers costs a full probe timeout, and 50 of them one after another + * would outlast the requester's wait. + */ +const PORT_PROBE_CONCURRENCY = 16; const MAX_KEEPALIVE_PORTS = config.fluxapps.maxAppsPerNode * MAX_TESTABLE_PORTS + NODE_SERVICE_PORTS_KEPT_ALIVE; /** @@ -651,15 +766,37 @@ async function checkAppAvailability(req, res) { // A requester that wants proof asks for it. One that does not - an older // node - gets exactly the check it always got. const echo = processedBody.echo === true; + // Every port read and answered for, a port that did not answer reading + // null, rather than the test ending at the first port that failed. Only + // with echo: the answers are what it returns. + const everyPort = echo && processedBody.everyPort === true; + // The protocols to test each port over. TCP alone when the caller names + // none; with both, UDP is probed once TCP has passed. + const protocols = processedBody.protocols ?? ['tcp']; + if (!Array.isArray(protocols) || !protocols.length || !protocols.every((protocol) => protocol === 'tcp' || protocol === 'udp')) { + throw new Error('Protocols to test must be tcp, udp or both'); + } + const tcp = protocols.includes('tcp'); + const udp = protocols.includes('udp'); const answered = {}; - // eslint-disable-next-line no-restricted-syntax - for (const port of ports) { + const testable = ports.filter((port) => { const iBP = isPortBanned(+port); const portNum = +port; const withinRange = portNum >= minPort && portNum <= maxPort; + if (!withinRange || iBP) { + log.error(`Flux App port ${port} is outside allowed range. minPort: ${minPort}, maxPort: ${maxPort}, isBanned: ${iBP}`); + } + return withinRange && !iBP; + }); - if (withinRange && !iBP) { + if (tcp && everyPort) { + await forEachLimited(testable, PORT_PROBE_CONCURRENCY, async (port) => { + answered[port] = await portAnswered(ip, port); + }); + } else if (tcp) { + // eslint-disable-next-line no-restricted-syntax + for (const port of testable) { if (echo) { // Read rather than merely reached. The requester compares. // eslint-disable-next-line no-await-in-loop @@ -675,17 +812,27 @@ async function checkAppAvailability(req, res) { throw new Error(`Flux Applications on ${ip}:${ipPort} are not available. Failed port: ${port}`); } } - } else { - log.error(`Flux App port ${port} is outside allowed range. minPort: ${minPort}, maxPort: ${maxPort}, isBanned: ${iBP}`); } } + + // A port this node could not send to is left out: that says nothing about + // the port, and the requester reads an absent port as no proof. + const udpAnswered = {}; + if (udp) { + await forEachLimited(testable, PORT_PROBE_CONCURRENCY, async (port) => { + const reply = await portAnsweredUdp(ip, +port); + if (reply !== undefined) udpAnswered[port] = reply; + }); + } const successResponse = messageHelper.createSuccessMessage(`Flux Applications on ${ip}:${ipPort} are available.`); // `answered` present at all is how the requester knows this peer READ the // ports rather than merely reaching them. Absent means no proof is // available from this peer, which is not the same as the ports being bad. // Added as a field rather than through createSuccessMessage, whose second // and third parameters are name and code. - if (echo) successResponse.data.answered = answered; + if (echo && tcp) successResponse.data.answered = answered; + // Present at all says this peer probed UDP, as `answered` does for TCP. + if (udp) successResponse.data.udpAnswered = udpAnswered; res.json(successResponse); } catch (error) { const errorResponse = messageHelper.createErrorMessage( @@ -844,10 +991,13 @@ async function keepUPNPPortsOpen(req, res) { * @param {string} value ip or ip:port to be set (normalized to ip:port) */ function setLocalSocketAddress(value) { + const wasKnown = localSocketAddressGate.ready; localSocketAddress = value ? normalizeSocketAddress(value) : null; // (Re)set the freshness deadline whenever the value changes, so getLocalSocketAddress // serves it without a benchmark RPC until it goes stale (cleared when set to null). localSocketAddressFreshUntil = localSocketAddress ? process.hrtime.bigint() + LOCAL_SOCKET_ADDRESS_TTL_NS : 0n; + if (localSocketAddress) localSocketAddressGate.open(); + else localSocketAddressGate.close(); // Told here because this is the one place the node learns what it is. The // peer manager needs it to keep this node out of its own peer draws - a node // that syncs from itself learns nothing, and it spends one of very few @@ -857,6 +1007,7 @@ function setLocalSocketAddress(value) { lastKnownSocketAddress = localSocketAddress; localSocketAddressEvents.emit('change', localSocketAddress); } + if (localSocketAddress && !wasKnown) localSocketAddressEvents.emit('known', localSocketAddress); } /** @@ -867,6 +1018,22 @@ function getKnownLocalSocketAddress() { return lastKnownSocketAddress; } +/** + * This node's socket address, once it has one. Resolves as soon as benchmark + * reports an address to any caller of getLocalSocketAddress; while none is known + * the own-standing watch keeps asking for one. + * @returns {Promise} ip:port + */ +async function waitForLocalSocketAddress() { + let address = await getLocalSocketAddress(); + while (!address) { + // eslint-disable-next-line no-await-in-loop + await localSocketAddressGate.wait(); + address = localSocketAddress; + } + return address; +} + /** * @param {function(string): void} listener Called with the new address (ip:port) * each time the node learns an address that differs from the last one. @@ -884,6 +1051,15 @@ function offLocalSocketAddressChange(listener) { localSocketAddressEvents.off('change', listener); } +/** + * @param {function(string): void} listener Called with the address each time the + * node has one after having none. + * @returns {void} + */ +function onLocalSocketAddressKnown(listener) { + localSocketAddressEvents.on('known', listener); +} + /** * Setter for dosMessage. * Main goal for this is testing availability. @@ -905,6 +1081,24 @@ function getDosMessage() { return getStickyDosMessage() || dosMessage; } +// Whether this node's ports answer over UDP, as the availability check last +// established it with a peer's proof or two peers' silence; null until it has. +let udpReachability = null; + +/** + * @param {{reachable: boolean, port?: number, peers?: string[]}|null} value + */ +function setUdpReachability(value) { + udpReachability = value; +} + +/** + * @returns {{reachable: boolean, port?: number, peers?: string[]}|null} + */ +function getUdpReachability() { + return udpReachability; +} + /** * Take this node out of service for as long as one owner says so. Idempotent * per owner. @@ -1029,6 +1223,7 @@ function setPlacementHold(owner, reason) { if (placementHolds.get(owner) === reason) return; placementHolds.set(owner, reason); log.info(`Placement hold set by ${owner}: ${reason}`); + fluxEventBus.publish('placementHold:set', { owner, reason }); } /** @@ -1042,6 +1237,7 @@ function clearPlacementHold(owner) { if (reason === undefined) return; placementHolds.delete(owner); log.info(`Placement hold cleared by ${owner} (was: ${reason})`); + fluxEventBus.publish('placementHold:cleared', { owner, reason }); } /** @@ -1299,6 +1495,79 @@ function checkDockerVersionAllowed(dockerVersion) { return false; } +/** + * Whether this node's cgroups are the ones FluxOS runs apps in: cgroup v2, + * a kernel that can kill a whole cgroup (cgroup.kill, Linux 5.14), and Docker + * placing containers through systemd. App slices, CPU burst and the kill a + * host stall makes all rely on the three. + * + * Asked once, when Docker first answers at startup. The kernel's answers are + * read from the cgroup filesystem: cgroup.kill is asked of FluxOS's own cgroup, + * which /proc/self/cgroup names and which is never the root, the one cgroup + * without the file. A driver that could not be read gives no verdict on the + * driver. + * @param {?string} cgroupDriver The cgroup driver Docker reports, or null when + * it could not be read. + * @param {{cgroupRoot?: string, selfCgroupFile?: string}} [paths] Where the + * cgroup filesystem is mounted, and the file naming this process's cgroup. + * @returns {Promise} True unless a requirement is known to be unmet. + */ +async function checkCgroupsAllowed(cgroupDriver, { cgroupRoot = '/sys/fs/cgroup', selfCgroupFile = '/proc/self/cgroup' } = {}) { + const exists = (file) => fs.access(path.join(cgroupRoot, file)).then(() => true, () => false); + // cgroup v2 lists one line, 0:: + const ownCgroup = (await fs.readFile(selfCgroupFile, 'utf8').catch(() => '')).match(/^0::(\/.*)$/m)?.[1]; + const unmet = []; + if (!await exists('cgroup.controllers')) unmet.push('cgroup v2'); + else if (!ownCgroup || !await exists(path.join(ownCgroup, 'cgroup.kill'))) unmet.push('cgroup.kill (Linux 5.14 or later)'); + if (cgroupDriver === null) { + log.error('checkCgroupsAllowed - Docker did not report its cgroup driver, no verdict on it'); + } else if (cgroupDriver !== 'systemd') { + unmet.push(`Docker's systemd cgroup driver (found ${cgroupDriver})`); + } + if (!unmet.length) return true; + setStickyDos( + StickyDosOwner.CGROUP_FLOOR, + `Cgroup Error. FluxOS requires cgroup v2, cgroup.kill and Docker's systemd cgroup driver; missing: ${unmet.join(', ')}. Fix the host and restart FluxOS`, + ); + return false; +} + +/** + * Whether the MongoDB this node runs meets the network minimum: the oldest + * release FluxOS supports. + * + * Asked once, when MongoDB first answers at startup. Upgrading MongoDB + * restarts mongod, not FluxOS, so the verdict stands until FluxOS next starts, + * and the message says so. A version that could not be read, or is not a + * SemVer version, gives no verdict. A pre-release is below its release. + * @param {?string} mongoDbVersion The version MongoDB reports, or null when it + * could not be read. + * @returns {boolean} True unless the version is known and below the floor. + */ +function checkMongoDbVersionAllowed(mongoDbVersion) { + const minimumVersion = config.minimumMongoDbAllowedVersion; + if (!minimumVersion) { + log.error('checkMongoDbVersionAllowed - no minimum MongoDB version configured, skipping the check'); + return true; + } + if (!mongoDbVersion) { + log.error('checkMongoDbVersionAllowed - MongoDB did not report its version, no verdict'); + return true; + } + if (!serviceHelper.parseSemver(mongoDbVersion)) { + log.error(`checkMongoDbVersionAllowed - MongoDB reported ${mongoDbVersion}, which is not a SemVer version, no verdict`); + return true; + } + if (serviceHelper.semverAtLeast(mongoDbVersion, minimumVersion)) { + return true; + } + setStickyDos( + StickyDosOwner.MONGODB_FLOOR, + `MongoDB Version Error. Current lower version allowed is v${minimumVersion} found v${mongoDbVersion}. Upgrade MongoDB and restart FluxOS`, + ); + return false; +} + /** * To check if Flux benchmark version is allowed. * @returns {boolean} True if version is verified as allowed. Otherwise false. @@ -2050,7 +2319,9 @@ async function fluxGoneFrom(socketAddress) { /** * Adopts `current` in place of `previous`, the configured address, which this * node has no record of leaving. Told to the network as a move only when this - * node ran apps there. + * node ran apps there. While that cannot be read nothing is adopted: the + * collision check that calls this finds the same listing on its next pass and + * asks again. * @param {string} previous * @param {string} current * @returns {Promise} @@ -2058,10 +2329,17 @@ async function fluxGoneFrom(socketAddress) { function leaveConfiguredAddress(previous, current) { return serialiseOwnAddressWork(async () => { const from = normalizeSocketAddress(previous); - await adjustExternalIp(from, current, { announceMove: await ranAppsAt(from) }); + const ran = await ranAppsAt(from); + if (ran === null) return; + await adjustExternalIp(from, current, { announceMove: ran }); }); } +// How long an adoption deferred on an unreadable database waits before this +// node's address is acted on again; the retry pending, while one is. +const OWN_ADDRESS_RETRY_MS = 60 * 1000; +let ownAddressRetry = null; + /** * Acts on the address benchmark reports as this node's own. One that differs * from the address the node last adopted is a move, whichever way it came and @@ -2071,6 +2349,8 @@ function leaveConfiguredAddress(previous, current) { * the Flux node there is gone (fluxGoneFrom) it adopts the reported address and * the collision check judges it; once it is, it adopts the reported address in * place of the configured one, told as a move only when it ran apps there. + * While whether it ran apps there cannot be read it adopts nothing, and acts on + * the address it knows again after OWN_ADDRESS_RETRY_MS. * @param {string} current The reported socket address. * @returns {Promise} */ @@ -2087,7 +2367,15 @@ function handleOwnAddress(current) { await writeOwnAddressRecord(reported); return; } - await adjustExternalIp(previous, reported, { announceMove: Boolean(record) || await ranAppsAt(previous) }); + const announceMove = record ? true : await ranAppsAt(previous); + if (announceMove === null) { + ownAddressRetry ??= setTimeout(() => { + ownAddressRetry = null; + handleOwnAddress(getKnownLocalSocketAddress() || reported); + }, OWN_ADDRESS_RETRY_MS); + return; + } + await adjustExternalIp(previous, reported, { announceMove }); } else if (!record) { await writeOwnAddressRecord(reported); } else { @@ -2115,6 +2403,8 @@ function startOwnAddressWatch() { return () => { if (ownAddressListener) offLocalSocketAddressChange(ownAddressListener); ownAddressListener = null; + clearTimeout(ownAddressRetry); + ownAddressRetry = null; }; } @@ -2844,36 +3134,42 @@ async function allowPort(port) { return cmdStat; } -// The ufw rule that admits IPv4 TCP clients to a port FluxOS opens for a moment, -// to test it from outside, and nothing else. ufw counts a rule that names no -// protocol as the same rule as the matching half of an app port's +// The ufw rule that admits IPv4 clients of one protocol to a port FluxOS opens +// for a moment, to test it from outside, and nothing else. ufw counts a rule that +// names no protocol as the same rule as the matching half of an app port's // `allow `, so a test that opened and deleted one would delete that half; -// a TCP rule is a rule of its own. -const testPortRule = (port) => ['proto', 'tcp', 'from', '0.0.0.0/0', 'to', 'any', 'port', String(port)]; +// a rule naming its protocol is a rule of its own. +const testPortRule = (port, protocol) => ['proto', protocol, 'from', '0.0.0.0/0', 'to', 'any', 'port', String(port)]; + +const TEST_PORT_PROTOCOLS = ['tcp', 'udp']; /** - * Opens a port to IPv4 TCP clients while FluxOS tests it from outside: the only rule - * for the port that deleteTestPortRule deletes. + * Opens a port to IPv4 clients of one protocol while FluxOS tests it from + * outside: the only rule for the port that deleteTestPortRule deletes. * @param {number|string} port Port. + * @param {string} [protocol] 'tcp' or 'udp'. * @returns {Promise<{status: boolean, message: (string|null)}>} */ -async function allowTestPort(port) { +async function allowTestPort(port, protocol = 'tcp') { if (Number.isNaN(+port)) return { status: false, message: 'Port needs to be a number' }; - const ran = await ufw.runUfw(['allow', ...testPortRule(port)]); + if (!TEST_PORT_PROTOCOLS.includes(protocol)) return { status: false, message: 'Protocol needs to be tcp or udp' }; + const ran = await ufw.runUfw(['allow', ...testPortRule(port, protocol)]); if (ran.locked) return { status: false, message: 'ufw is locked by another ufw command' }; const cmdres = ran.stdout + ran.stderr; return { status: /added|updated|existing/.test(cmdres), message: cmdres }; } /** - * Deletes the rule allowTestPort writes, and no other rule for the port. A rule - * already gone counts as deleted. + * Deletes the rule allowTestPort writes for a protocol, and no other rule for + * the port. A rule already gone counts as deleted. * @param {number|string} port Port. + * @param {string} [protocol] 'tcp' or 'udp'. * @returns {Promise<{status: boolean, message: (string|null)}>} */ -async function deleteTestPortRule(port) { +async function deleteTestPortRule(port, protocol = 'tcp') { if (Number.isNaN(+port)) return { status: false, message: 'Port needs to be a number' }; - const ran = await ufw.runUfw(['delete', 'allow', ...testPortRule(port)]); + if (!TEST_PORT_PROTOCOLS.includes(protocol)) return { status: false, message: 'Protocol needs to be tcp or udp' }; + const ran = await ufw.runUfw(['delete', 'allow', ...testPortRule(port, protocol)]); if (ran.locked) return { status: false, message: 'ufw is locked by another ufw command' }; const cmdres = ran.stdout + ran.stderr; return { status: cmdres.includes('delete'), message: cmdres }; @@ -3465,8 +3761,10 @@ module.exports = { checkFluxAvailability, getLocalSocketAddress, getKnownLocalSocketAddress, + waitForLocalSocketAddress, onLocalSocketAddressChange, offLocalSocketAddressChange, + onLocalSocketAddressKnown, getFluxNodePrivateKey, getFluxNodePublicKey, MAX_KEEPALIVE_PORTS, @@ -3500,6 +3798,8 @@ module.exports = { checkFluxbenchVersionAllowed, checkNodeJsVersionAllowed, checkDockerVersionAllowed, + checkCgroupsAllowed, + checkMongoDbVersionAllowed, checkMyFluxAvailability, adjustExternalIp, handleOwnAddress, @@ -3523,6 +3823,8 @@ module.exports = { setLocalSocketAddress, getDosMessage, setDosMessage, + getUdpReachability, + setUdpReachability, setDosStateValue, getDosStateValue, isNodeDos, @@ -3543,6 +3845,7 @@ module.exports = { isPortOpen, portAnswered, MAX_ECHO_BYTES, + portAnsweredUdp, checkAppAvailability, verifySignedFluxnodeMessage, isPortEnterprise, diff --git a/ZelBack/src/services/fluxService.js b/ZelBack/src/services/fluxService.js index 573ffb2297..94e843c3af 100644 --- a/ZelBack/src/services/fluxService.js +++ b/ZelBack/src/services/fluxService.js @@ -1532,7 +1532,11 @@ async function getFluxInfo(req, res) { benchmark: {}, flux: {}, apps: {}, - geolocation: await geolocationService.getNodeGeolocation(), + // null while this process knows no geolocation. + geolocation: await geolocationService.getNodeGeolocation().catch((error) => { + log.warn(`flux info - geolocation unreadable: ${error.message}`); + return null; + }), }; const versionRes = await getFluxVersion(); if (versionRes.status === 'error') { @@ -1658,9 +1662,6 @@ async function getFluxInfo(req, res) { // at one exit and not the other is the shape it exists to avoid. info.apps.runningapps = appQueryService.publicContainerView(appsRunning.data); const appsResources = await resourceQueryService.appsResources(); - if (appsResources.status === 'error') { - throw appsResources.data; - } // The same three numbers /apps/appsresources publishes. This endpoint embeds // them, and a field kept from one exit and not the other is the shape that // catches this file out. diff --git a/ZelBack/src/services/geolocationService.js b/ZelBack/src/services/geolocationService.js index 723dd7b9c5..c195fca9cb 100644 --- a/ZelBack/src/services/geolocationService.js +++ b/ZelBack/src/services/geolocationService.js @@ -80,52 +80,37 @@ async function storeGeolocationToDb(geolocation, isStaticIp, isDataCenter, ipCha } /** - * Retrieves geolocation data from the database + * Retrieves geolocation data from the database. Rejects when it cannot be read. * @returns {Promise<{geolocation: object|null, staticIp: boolean, dataCenter: boolean, lastIpChangeDate: number|null}>} */ async function getGeolocationFromDb() { - try { - const dbClient = dbHelper.databaseConnection(); - if (!dbClient) { - return { geolocation: null, staticIp: false, dataCenter: false, lastIpChangeDate: null }; - } - const database = dbClient.db(config.database.local.database); - const query = { _id: 'nodeGeolocation' }; - const result = await dbHelper.findOneInDatabase(database, geolocationCollection, query); - if (result && result.geolocation) { - return { - geolocation: result.geolocation, - staticIp: result.staticIp || false, - dataCenter: result.dataCenter || false, - lastIpChangeDate: result.lastIpChangeDate || null, - ipFirstSeenAt: result.ipFirstSeenAt || null, - // `?? null`: a document written before these fields existed carries no - // observation, and saying so lets the next pass start the window. - staticIpState: result.staticIpState ?? null, - networkEvidence: result.networkEvidence ?? null, - }; - } - return { - geolocation: null, - staticIp: false, - dataCenter: false, - lastIpChangeDate: null, - ipFirstSeenAt: null, - staticIpState: null, - networkEvidence: null, - }; - } catch (error) { - log.error(`Failed to retrieve geolocation from database: ${error.message}`); + const dbClient = dbHelper.databaseConnection(); + if (!dbClient) throw new Error('no database connection'); + const database = dbClient.db(config.database.local.database); + const query = { _id: 'nodeGeolocation' }; + const result = await dbHelper.findOneInDatabase(database, geolocationCollection, query); + if (result && result.geolocation) { return { - geolocation: null, - staticIp: false, - dataCenter: false, - lastIpChangeDate: null, - ipFirstSeenAt: null, - staticIpState: null, - networkEvidence: null, + geolocation: result.geolocation, + staticIp: result.staticIp || false, + dataCenter: result.dataCenter || false, + lastIpChangeDate: result.lastIpChangeDate || null, + ipFirstSeenAt: result.ipFirstSeenAt || null, + // `?? null`: a document written before these fields existed carries no + // observation, and saying so lets the next pass start the window. + staticIpState: result.staticIpState ?? null, + networkEvidence: result.networkEvidence ?? null, }; } + return { + geolocation: null, + staticIp: false, + dataCenter: false, + lastIpChangeDate: null, + ipFirstSeenAt: null, + staticIpState: null, + networkEvidence: null, + }; } /** @@ -559,7 +544,8 @@ async function setNodeGeolocation() { * the node's own record of where it is, and the next refresh persists whatever * it holds. The record is flat scalars, so one level copies all of it - a * nested field added later needs this to deepen with it. - * @returns {Promise} The geolocation object or null + * @returns {Promise} The geolocation object, or null when none is + * known. Rejects when the stored record is needed and cannot be read. */ async function getNodeGeolocation() { if (storedGeolocation) { diff --git a/ZelBack/src/services/hostStallService.js b/ZelBack/src/services/hostStallService.js new file mode 100644 index 0000000000..b2d49af3b4 --- /dev/null +++ b/ZelBack/src/services/hostStallService.js @@ -0,0 +1,382 @@ +/** + * Take a node out of service when a thread of one of its dependencies is held + * in the kernel for good. + * + * WHAT COUNTS. A thread in uninterruptible sleep (state D) whose context switch + * count is the same at two readings never ran between them, however far apart + * they are. A thread that stays so for one whole interval holds the node back + * from new placements; for STUCK_INTERVALS intervals, the host is broken, and + * the node goes out of service until it reboots. + * + * Readings are counted rather than time measured: a VM its hypervisor paused + * takes no readings while it is paused, so a pause cannot read as a stall. A + * reading counts only when one interval has passed, by the machine's uptime, + * since the reading the thread's count last moved at: FluxOS takes a reading as + * it starts, and a start soon after another adds nothing. + * + * Out of service lasts until the machine reboots, through any number of FluxOS + * restarts, and so does the count of readings toward it. Both are kept in + * /run/fluxos, on the tmpfs every Linux mounts at /run at boot: memory, so + * reading it cannot stall with a disk, and emptied by a reboot. Only root and + * FluxOS's own account may write there. Not a database row, because the database + * may be what is stuck. The file is written only when what it holds changes. + * + * WHAT IS READ. The threads of the processes FluxOS depends on, in FluxOS's own + * PID namespace. An app container that runs one of the same programs has its + * own namespace and is not read. Every file read is one any account may read of + * another's process, so a FluxOS that is not root reads the same. + */ + +const fs = require('fs').promises; +const os = require('os'); +const path = require('path'); +const config = require('config'); +const log = require('../lib/log'); +const serviceHelper = require('./serviceHelper'); +const fluxNetworkHelper = require('./fluxNetworkHelper'); + +const DEPENDENCIES = new Set(['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd']); +const READ_INTERVAL_MS = config.hostStall.readIntervalMs; +const STUCK_INTERVALS = config.hostStall.stuckIntervals; +// /proc/uptime is written in hundredths of a second. +const UPTIME_RESOLUTION_MS = 10; + +/** + * Every dependency thread in D at the last reading, keyed by process, thread + * and the thread's start time so a reused id is a new thread, with its count + * and the uptime its count last moved at. + * @type {Map} + */ +let tracked = new Map(); +/** Why the node is out of service for a host stall, once it is. */ +let tripReason = null; +let running = false; +let readTimer = null; +// Advanced by stop(), so a reading still under way when the service stops +// neither keeps what it read nor acts on it. +let generation = 0; +/** The state being kept, which stop() lets its caller wait for. */ +let keeping = Promise.resolve(); +/** Called once the node is out of service for a host stall. */ +const tripListeners = []; +const DEFAULT_PATHS = { + procRoot: '/proc', + stateFile: '/run/fluxos/host-stall.json', +}; +let paths = { ...DEFAULT_PATHS }; +// What the state file holds with nothing stuck and no trip. +const NOTHING_KEPT = JSON.stringify({ trip: null, threads: [] }); +/** What the state file holds, as this process last wrote or found it; null when unknown. */ +let keptText = NOTHING_KEPT; +/** What the state file is to hold once the write asked for last has landed; null when none is waiting. */ +let queuedText = null; + +/** + * @param {{procRoot?: string, stateFile?: string}} options + * Where /proc and the kept state are; each one given replaces it. + */ +function usePaths(options) { + paths = { ...paths, ...options }; +} + +/** + * @param {string} file + * @returns {Promise} The file's text, or null when it is gone. + */ +async function readOrNull(file) { + return fs.readFile(file, 'utf8').catch(() => null); +} + +/** + * One thread's state, start time, context switches and wait point, or null + * when it exited while it was read. + * @param {string} taskDir /proc//task/ + */ +async function readThread(taskDir) { + const [stat, status, wchan] = await Promise.all(['stat', 'status', 'wchan'].map((name) => readOrNull(path.join(taskDir, name)))); + if (stat === null || status === null) return null; + // the command name inside the parentheses may itself hold spaces or ')' + const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' '); + const switches = status.split('\n') + .filter((line) => line.startsWith('voluntary_ctxt_switches') || line.startsWith('nonvoluntary_ctxt_switches')) + .reduce((sum, line) => sum + Number(line.split(/\s+/)[1]), 0); + // the kernel writes 0 for a wait point it does not show this account + const waitPoint = (wchan ?? '').trim(); + return { + state: fields[0], startTime: fields[19], switches, wchan: waitPoint && waitPoint !== '0' ? waitPoint : 'unknown', + }; +} + +/** + * How many PID namespaces deep a process is, counted from the one this /proc + * shows: its status names its id in each, outermost first. A process in a + * namespace below FluxOS's has more than FluxOS; one beside it is not in this + * /proc at all. A kernel older than 4.1 writes no NSpid, and every process on it + * reads as one deep. + * @param {string|null} status The process's status file. + * @returns {number|null} Null when the status is gone. + */ +function namespaceDepth(status) { + if (status === null) return null; + const line = status.split('\n').find((entry) => entry.startsWith('NSpid:')); + return line ? line.trim().split(/\s+/).length - 1 : 1; +} + +/** + * Every thread of every dependency process in this PID namespace. + * @param {string} procRoot The /proc this reading reads, fixed when it starts. + * @returns {Promise>} + */ +async function readDependencyThreads(procRoot) { + const ownDepth = namespaceDepth(await readOrNull(path.join(procRoot, 'self', 'status'))); + const pids = (await fs.readdir(procRoot)).filter((entry) => /^\d+$/.test(entry)); + const threads = []; + // eslint-disable-next-line no-restricted-syntax + for (const pid of pids) { + const pidDir = path.join(procRoot, pid); + // eslint-disable-next-line no-await-in-loop + const comm = (await readOrNull(path.join(pidDir, 'comm')))?.trim(); + // eslint-disable-next-line no-continue + if (!DEPENDENCIES.has(comm)) continue; + // eslint-disable-next-line no-await-in-loop + const depth = namespaceDepth(await readOrNull(path.join(pidDir, 'status'))); + // eslint-disable-next-line no-continue + if (depth === null || depth !== ownDepth) continue; + // eslint-disable-next-line no-await-in-loop + const tids = await fs.readdir(path.join(pidDir, 'task')).catch(() => []); + // eslint-disable-next-line no-restricted-syntax + for (const tid of tids) { + // eslint-disable-next-line no-await-in-loop + const thread = await readThread(path.join(pidDir, 'task', tid)); + if (thread) threads.push({ process: comm, pid, tid, ...thread }); + } + } + return threads; +} + +/** + * The machine's uptime, which runs on through FluxOS restarts and starts again + * only at a boot. + * @param {string} procRoot + * @returns {Promise} Milliseconds, or null when it cannot be read. + */ +async function readUptimeMs(procRoot) { + const seconds = Number((await readOrNull(path.join(procRoot, 'uptime')))?.split(' ')[0]); + return Number.isFinite(seconds) ? seconds * 1000 : null; +} + +/** + * Carry each thread's count of intervals stuck from the last reading to this + * one. A thread that did not run adds one once an interval has passed since its + * count last moved, and keeps its count before then. A thread that ran, or left + * D, starts again. + * @param {Array} threads One reading. + * @param {number} nowMs The uptime the reading was taken at. + * @returns {Map} + */ +function carryForward(threads, nowMs) { + const next = new Map(); + threads.filter((thread) => thread.state === 'D').forEach((thread) => { + const key = `${thread.process}/${thread.pid}/${thread.tid}/${thread.startTime}`; + const before = tracked.get(key); + let intervals = 0; + let countedAtMs = nowMs; + if (before && before.switches === thread.switches) { + const intervalPassed = nowMs - before.countedAtMs + UPTIME_RESOLUTION_MS >= READ_INTERVAL_MS; + intervals = intervalPassed ? before.intervals + 1 : before.intervals; + countedAtMs = intervalPassed ? nowMs : before.countedAtMs; + } + next.set(key, { + process: thread.process, tid: thread.tid, wchan: thread.wchan, switches: thread.switches, intervals, countedAtMs, + }); + }); + return next; +} + +/** + * @param {{process: string, tid: string, wchan: string, intervals: number}} thread + * @returns {string} + */ +function describe(thread) { + const minutes = Math.round((thread.intervals * READ_INTERVAL_MS) / 60000); + return `${thread.process} thread ${thread.tid} has been held in the kernel at ${thread.wchan} for ${minutes} minutes`; +} + +/** + * Make the state file's directory. /run is root's: a FluxOS that is not root + * makes it through sudo, owned by its own account, and writes in it directly. + * @param {string} dir + */ +async function makeStateDirectory(dir) { + try { + await fs.mkdir(dir, { recursive: true, mode: 0o755 }); + return; + } catch (error) { + if (error.code !== 'EACCES' && error.code !== 'EPERM') throw error; + } + const { uid, gid } = os.userInfo(); + const { error } = await serviceHelper.runCommand('install', { + runAsRoot: true, + params: ['-d', '-m', '755', '-o', String(uid), '-g', String(gid), dir], + }); + if (error) throw error; +} + +/** + * Write the state file, once every write asked for before has ended. + * @param {string} text + * @param {string} stateFile The file as it was when the write was asked for. + */ +async function writeState(text, stateFile) { + const staged = `${stateFile}.tmp`; + try { + await makeStateDirectory(path.dirname(stateFile)); + await fs.writeFile(staged, text); + await fs.rename(staged, stateFile); + keptText = text; + } catch (error) { + log.error(`hostStall - the state could not be kept: ${error.message}`); + } finally { + if (queuedText === text) queuedText = null; + } +} + +/** + * Keep the trip and every thread's count, so a FluxOS restart before the next + * reboot carries both on. Nothing is written when the file already holds them, or + * a write waiting to land does. Writes land one at a time and in order, and no + * reading waits for one. + */ +function keepState() { + const trip = fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL) ? tripReason : null; + const text = JSON.stringify({ trip, threads: [...tracked] }); + if (text === (queuedText ?? keptText)) return; + queuedText = text; + const { stateFile } = paths; + keeping = keeping.then(() => writeState(text, stateFile)); +} + +/** Tell every listener the node is out of service for a host stall. */ +function announceTrip() { + tripListeners.forEach((listener) => listener()); +} + +/** + * Call `listener` once the node is out of service for a host stall: at once + * when it already is, whether it tripped in this process or the trip was + * restored at start. + * @param {function(): void} listener + */ +function onTrip(listener) { + tripListeners.push(listener); + if (fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) listener(); +} + +/** Restore what an earlier FluxOS in this boot kept: the trip, and every thread's count. */ +async function restoreState() { + const text = await readOrNull(paths.stateFile); + keptText = text === null ? NOTHING_KEPT : null; + if (text === null) return; + try { + const { trip, threads } = JSON.parse(text); + tracked = new Map(threads); + keptText = text; + if (trip) { + tripReason = trip; + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, trip); + announceTrip(); + } + } catch (error) { + log.error(`hostStall - the kept state could not be read: ${error.message}`); + } +} + +/** Hold or release the node by the longest-stuck thread. */ +async function act() { + const longest = [...tracked.values()].reduce((worst, thread) => (!worst || thread.intervals > worst.intervals ? thread : worst), null); + if (!longest || longest.intervals === 0) { + fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL); + return; + } + fluxNetworkHelper.setPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL, describe(longest)); + if (longest.intervals >= STUCK_INTERVALS && !fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL)) { + tripReason = `Host stall: ${describe(longest)}`; + fluxNetworkHelper.setStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL, tripReason); + announceTrip(); + } +} + +/** Take one reading, act on it, and ask for what it found to be kept. */ +async function takeReading() { + const startedIn = generation; + const nowMs = await readUptimeMs(paths.procRoot); + if (nowMs === null) { + log.error('hostStall - the uptime could not be read; this reading is not taken'); + return; + } + const threads = await readDependencyThreads(paths.procRoot); + if (startedIn !== generation) return; + tracked = carryForward(threads, nowMs); + await act(); + keepState(); +} + +/** + * Take one reading and act on it. + * @param {object} [options] Paths, as usePaths takes them. + * @returns {Promise} Resolves once what it found is kept. + */ +async function readOnce(options = {}) { + usePaths(options); + await takeReading(); + await keeping; +} + +async function readAndReschedule() { + try { + await takeReading(); + } catch (error) { + log.error(`hostStall - reading the dependency threads failed: ${error.message}`); + } + if (running) readTimer = setTimeout(readAndReschedule, READ_INTERVAL_MS); +} + +/** + * Restore what an earlier FluxOS in this boot kept, then read now, and every + * READ_INTERVAL_MS after each reading ends. + * @param {object} [options] Paths, as usePaths takes them. + * @returns {Promise} Resolves once the kept state is restored and the first + * reading has acted; the write of what it found goes on without it. + */ +async function start(options = {}) { + if (running) return; + running = true; + usePaths(options); + await restoreState(); + await readAndReschedule(); +} + +/** + * Take no more readings and forget every thread. + * @returns {Promise} Resolves once the state being kept, if any, is written. + */ +function stop() { + generation += 1; + running = false; + clearTimeout(readTimer); + readTimer = null; + tracked = new Map(); + tripReason = null; + keptText = NOTHING_KEPT; + queuedText = null; + paths = { ...DEFAULT_PATHS }; + tripListeners.length = 0; + return keeping; +} + +module.exports = { + start, + stop, + onTrip, + readOnce, +}; diff --git a/ZelBack/src/services/idService.js b/ZelBack/src/services/idService.js index 365262108d..ab719696cb 100644 --- a/ZelBack/src/services/idService.js +++ b/ZelBack/src/services/idService.js @@ -1,3 +1,4 @@ +const { EventEmitter } = require('node:events'); const config = require('config'); const qs = require('qs'); const log = require('../lib/log'); @@ -26,17 +27,65 @@ const PRIVILEGE_RESPONSE = Object.freeze({ NONE: 'none', }); -async function deleteLoginPhrase(phrase) { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.local.database); - const collection = config.database.local.collections.activeLoginPhrases; - const query = { loginPhrase: phrase }; - const projection = {}; - await dbHelper.findOneAndDeleteInDatabase(database, collection, query, projection); - } catch (error) { - log.error(error); +// How long a login phrase is valid for once issued; the activeLoginPhrases TTL +// index removes it then. +const LOGIN_PHRASE_VALID_MS = 15 * 60 * 1000; + +// The outcome of a login, published by verifyLogin under the phrase it used: +// { data } with what the login answered, or { error } once the phrase is used +// and the login was not written. The login websocket waits on it. +const loginOutcomes = new EventEmitter(); +loginOutcomes.setMaxListeners(0); +// For each phrase, how many logins are between trying to use it and ending. A login +// is counted before it tries the phrase and until after it has published how it +// ended, so a used phrase nobody is counted against has ended without a login or +// with one already written. +const loginsInProgress = new Map(); +// A phrase not yet used by a login. +const UNUSED_PHRASE = { usedAt: { $exists: false } }; + +function enterLogin(phrase) { + loginsInProgress.set(phrase, (loginsInProgress.get(phrase) ?? 0) + 1); +} + +function leaveLogin(phrase) { + const held = loginsInProgress.get(phrase); + if (held > 1) { + loginsInProgress.set(phrase, held - 1); + return; } + loginsInProgress.delete(phrase); + // a page waiting on the phrase reads again: no login is left to publish anything + loginOutcomes.emit(phrase, { ended: true }); +} + +/** + * Publish how a login that used its phrase ended. + * @param {string} phrase + * @param {{data?: object, error?: object}} outcome + */ +function publishLoginOutcome(phrase, outcome) { + loginOutcomes.emit(phrase, outcome); +} + +/** + * Marks a login phrase used, so that it signs in once: of the logins that try it, + * one marks it. The record stays until its TTL removes it, so a login page reads + * a used phrase as used rather than as never issued. + * @param {string} phrase + * @returns {Promise} Whether this call marked it. Rejects when the + * database does not answer. + */ +async function claimLoginPhrase(phrase) { + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.local.database); + const collection = config.database.local.collections.activeLoginPhrases; + return Boolean(await dbHelper.findOneAndUpdateInDatabase( + database, + collection, + { loginPhrase: phrase, ...UNUSED_PHRASE }, + { $set: { usedAt: new Date() } }, + )); } @@ -59,7 +108,7 @@ async function loginPhrase(req, res) { const collection = config.database.local.collections.activeLoginPhrases; const timestamp = Date.now(); - const validTill = timestamp + (15 * 60 * 1000); // 15 minutes + const validTill = timestamp + LOGIN_PHRASE_VALID_MS; const phrase = timestamp + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); const newLoginPhrase = { @@ -87,7 +136,7 @@ async function loginPhrase(req, res) { async function emergencyPhrase(req, res) { try { const timestamp = Date.now(); - const validTill = timestamp + (15 * 60 * 1000); // 15 minutes + const validTill = timestamp + LOGIN_PHRASE_VALID_MS; const phrase = timestamp + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15) + Math.random().toString(36).substring(2, 15); const db = dbHelper.databaseConnection(); @@ -121,6 +170,9 @@ async function verifyLogin(req, res) { body += data; }); req.on('end', async () => { + // set once this request has used the phrase, which no other can then use + let usedPhrase = null; + let triedPhrase = null; try { const processedBody = serviceHelper.ensureObject(body); const address = processedBody.zelid || processedBody.address; @@ -157,7 +209,7 @@ async function verifyLogin(req, res) { const db = dbHelper.databaseConnection(); const database = db.db(config.database.local.database); const collection = config.database.local.collections.activeLoginPhrases; - const query = { loginPhrase: message }; + const query = { loginPhrase: message, ...UNUSED_PHRASE }; const projection = {}; const result = await dbHelper.findOneInDatabase(database, collection, query, projection); @@ -196,6 +248,14 @@ async function verifyLogin(req, res) { } else if (address === adminZelid) { privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } + // A phrase signs in once: the login is written only by the request that + // marked it used. + triedPhrase = message; + enterLogin(message); + if (!await claimLoginPhrase(message)) { + throw new Error('Signed message is no longer valid. Please request a new one.'); + } + usedPhrase = message; // The signature is the session's credential. The login websocket hands it // to whoever holds the login phrase, so it is kept apart from the login, // in a collection whose TTL index expires it a minute after the login. It @@ -218,8 +278,8 @@ async function verifyLogin(req, res) { expireAt, }; const resMessage = messageHelper.createDataMessage(resData); + publishLoginOutcome(message, { data: resData }); res.json(resMessage); - deleteLoginPhrase(message); } else { throw new Error('Invalid signature'); } @@ -232,7 +292,10 @@ async function verifyLogin(req, res) { } catch (error) { log.error(error); const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); + if (usedPhrase) publishLoginOutcome(usedPhrase, { error: errMessage }); res.json(errMessage); + } finally { + if (triedPhrase) leaveLogin(triedPhrase); } }); } @@ -312,7 +375,7 @@ async function activeLoginPhrases(req, res) { const database = db.db(config.database.local.database); const collection = config.database.local.collections.activeLoginPhrases; - const query = {}; + const query = UNUSED_PHRASE; const projection = { projection: { _id: 0, loginPhrase: 1, createdAt: 1, expireAt: 1, @@ -526,102 +589,118 @@ async function logoutAllUsers(req, res) { } /** - * To check if a login phrase is currently active. The user's Flux ID, login phrase, signature and privilege level are returned if the login phrase is active. + * Answers a login page waiting on its phrase: with the login once the phrase signs + * in, or with why it no longer can. The answer is the outcome verifyLogin + * publishes for the phrase; a page that connects after the login reads it instead. + * A page waiting on a used phrase reads again once no login is counted against it. * @param {object} ws Web socket. - * @param {object} req Request. + * @param {string} loginphrase */ async function wsRespondLoginPhrase(ws, loginphrase) { - // console.log(loginphrase) - // respond with object containing address and signature to received message - let connclosed = false; - // eslint-disable-next-line no-param-reassign - ws.onclose = (evt) => { - console.log(evt.code); - connclosed = true; - }; - // eslint-disable-next-line no-param-reassign - ws.onerror = (evt) => { - log.error(evt.code); - connclosed = true; - }; - const db = dbHelper.databaseConnection(); - - const database = db.db(config.database.local.database); - const collection = config.database.local.collections.loggedUsers; + let settled = false; + let expiry = null; + const noLongerValid = () => messageHelper.createErrorMessage('Signed message is no longer valid. Please request a new one.'); + const database = dbHelper.databaseConnection().db(config.database.local.database); const query = { loginPhrase: loginphrase }; - const projection = {}; - // eslint-disable-next-line no-inner-declarations - async function searchDatabase() { + + function stop() { + settled = true; + loginOutcomes.removeListener(loginphrase, onOutcome); + clearTimeout(expiry); + } + function finish(message, code) { + if (settled) return; + stop(); try { - const result = await dbHelper.findOneInDatabase(database, collection, query, projection).catch((error) => { - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - ws.send(qs.stringify(errMessage)); - ws.close(4011); - throw error; - }); - if (result) { - // user is logged, all ok - const loginSignature = await dbHelper.findOneInDatabase(database, config.database.local.collections.loginSignatures, query, projection); + ws.send(qs.stringify(message)); + ws.close(code); + } catch (error) { + log.error(error); + } + } + + // Reads what the phrase has come to and answers it, or waits for the login that + // is to use it. + async function decide() { + let login; + try { + login = await dbHelper.findOneInDatabase(database, config.database.local.collections.loggedUsers, query, {}); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); + return; + } + if (settled) return; + if (login) { + try { + const loginSignature = await dbHelper.findOneInDatabase(database, config.database.local.collections.loginSignatures, query, {}); const adminZelid = verificationHelperUtils.nodeOperatorZelid(); if (!adminZelid) { throw new Error('Node is still starting and cannot establish privileges yet'); } let privilage = PRIVILEGE_RESPONSE.USER; - if (result.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(result.zelid)) { + if (login.zelid === config.fluxTeamFluxID || verificationHelperUtils.isFluxSupportTeamZelid(login.zelid)) { privilage = PRIVILEGE_RESPONSE.FLUX_TEAM; - } else if (result.zelid === adminZelid) { + } else if (login.zelid === adminZelid) { privilage = PRIVILEGE_RESPONSE.NODE_OPERATOR; } - const resData = { + finish(messageHelper.createDataMessage({ message: 'Successfully logged in', - zelid: result.zelid, - loginPhrase: result.loginPhrase, + zelid: login.zelid, + loginPhrase: login.loginPhrase, signature: loginSignature ? loginSignature.signature : undefined, privilage, - createdAt: result.createdAt, - expireAt: result.expireAt, - }; - const message = messageHelper.createDataMessage(resData); - if (!connclosed) { - try { - ws.send(qs.stringify(message)); - ws.close(4012); - } catch (e) { - log.error(e); - } - } - } else { - // check if this loginPhrase is still active. If so rerun this searching process - const activeLoginPhrasesCollection = config.database.local.collections.activeLoginPhrases; - const resultB = await dbHelper.findOneInDatabase(database, activeLoginPhrasesCollection, query, projection).catch((error) => { - const errMessage = messageHelper.createErrorMessage(error.message, error.name, error.code); - ws.send(qs.stringify(errMessage)); - ws.close(4013); - throw error; - }); - if (resultB) { - setTimeout(() => { - if (!connclosed) { - searchDatabase(); - } - }, 500); - } else { - const errMessage = messageHelper.createErrorMessage('Signed message is no longer valid. Please request a new one.'); - if (!connclosed) { - try { - ws.send(qs.stringify(errMessage)); - ws.close(); - } catch (e) { - log.error(e); - } - } - } + createdAt: login.createdAt, + expireAt: login.expireAt, + }), 4012); + } catch (error) { + log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4011); } + return; + } + + let phrase; + try { + phrase = await dbHelper.findOneInDatabase(database, config.database.local.collections.activeLoginPhrases, query, {}); } catch (error) { log.error(error); + finish(messageHelper.createErrorMessage(error.message, error.name, error.code), 4013); + return; } + if (settled) return; + // A phrase never issued, or expired, no login will use. A used one with no login + // counted against it ended without the login read above. A login still counted + // publishes how it ends. + if (!phrase || (phrase.usedAt && !loginsInProgress.has(loginphrase))) { + finish(noLongerValid()); + return; + } + if (expiry) return; + const issuedAt = +loginphrase.substring(0, 13); + expiry = setTimeout(() => finish(noLongerValid()), Math.max(issuedAt + LOGIN_PHRASE_VALID_MS - Date.now(), 0)); + if (expiry.unref) expiry.unref(); } - searchDatabase(); + + function onOutcome({ data, error, ended }) { + if (ended) { + if (!settled) decide(); + return; + } + if (data) finish(messageHelper.createDataMessage(data), 4012); + else finish(error); + } + + // eslint-disable-next-line no-param-reassign + ws.onclose = stop; + // eslint-disable-next-line no-param-reassign + ws.onerror = (evt) => { + log.error(evt.code); + stop(); + }; + // before the reads, so a login that finishes while they run is not missed + loginOutcomes.on(loginphrase, onOutcome); + await decide(); } /** diff --git a/ZelBack/src/services/migrations/paymentPositions.js b/ZelBack/src/services/migrations/paymentPositions.js index 26b29171c9..f4d5418258 100644 --- a/ZelBack/src/services/migrations/paymentPositions.js +++ b/ZelBack/src/services/migrations/paymentPositions.js @@ -6,12 +6,23 @@ const daemonServiceUtils = require('../daemonService/daemonServiceUtils'); const messageVerifier = require('../appMessaging/messageVerifier'); const { appPaymentPositions } = require('../appPaymentPositions'); +// Records written per round trip: each batch is one bulkWrite, so a fleet's +// worth of records never rides on a single call. +const BACKFILL_BATCH = 5000; + +// The payment records the backfill reads: those without a position. A record +// already marked notOnChain is decided: if its transaction is mined later, the +// explorer records the payment from that block. Read through the collection's +// txIndex index (explorerService.prepareCollections), so a node whose records all +// have positions reads none of them. +const UNPOSITIONED = { txIndex: { $exists: false }, notOnChain: { $ne: true } }; + /** * Gives every app payment record without a position in its block the position * and the height the daemon's address index holds for its transaction: one call * over the payment addresses from the first height a payment is recorded at to * the chain tip, so a transaction the index does not hold is not on the chain. - * Such a record is marked `notOnChain`, and is no payment. A record above the + * Such a record is marked `notOnChain`, is no payment, and is not read again. A record above the * daemon's tip is left for a later run: the daemon has not reached its block. * * The height is the chain's, not the one recorded: a record written from a block @@ -21,13 +32,15 @@ const { appPaymentPositions } = require('../appPaymentPositions'); * them again at the next start; the registry row of its app is written after * them, since which message governs reads their positions * (messageVerifier.moveStoredMessagesToPayments, storeGoverningAppSpecifications). + * The records are written BACKFILL_BATCH at a time; a run that stops part way + * leaves the rest without a position, and the next run takes them. * @returns {Promise<{positioned: number, heightsCorrected: number, notOnChain: number, aboveTip: number}>} */ async function backfillPaymentPositions() { const database = dbHelper.databaseConnection().db(config.database.daemon.database); const collection = config.database.daemon.collections.appsHashes; const missing = await dbHelper.findInDatabase( - database, collection, { txIndex: { $exists: false } }, { + database, collection, UNPOSITIONED, { projection: { _id: 0, hash: 1, txid: 1, height: 1, value: 1, }, @@ -72,7 +85,10 @@ async function backfillPaymentPositions() { } } const moved = await messageVerifier.moveStoredMessagesToPayments(corrected); - if (operations.length) await dbHelper.bulkWriteInDatabase(database, collection, operations); + for (let i = 0; i < operations.length; i += BACKFILL_BATCH) { + // eslint-disable-next-line no-await-in-loop + await dbHelper.bulkWriteInDatabase(database, collection, operations.slice(i, i + BACKFILL_BATCH)); + } await messageVerifier.storeGoverningAppSpecifications(moved); log.info(`migrations - payment positions: ${operations.length - notOnChain} positioned, ${heightsCorrected} of them at the chain's height rather than the recorded one, ${notOnChain} not on the chain, ${aboveTip} above the daemon's tip`); return { @@ -80,4 +96,4 @@ async function backfillPaymentPositions() { }; } -module.exports = { backfillPaymentPositions }; +module.exports = { backfillPaymentPositions, BACKFILL_BATCH, UNPOSITIONED }; diff --git a/ZelBack/src/services/migrations/placements.js b/ZelBack/src/services/migrations/placements.js new file mode 100644 index 0000000000..9eff23ab01 --- /dev/null +++ b/ZelBack/src/services/migrations/placements.js @@ -0,0 +1,27 @@ +const config = require('config'); +const dbHelper = require('../dbHelper'); +const fluxNetworkHelper = require('../fluxNetworkHelper'); +const appPlacements = require('../appManagement/appPlacements'); +const { localAppsInformation } = require('../utils/appConstants'); +const { untilSucceeds } = require('../utils/untilSucceeds'); + +/** + * Gives each app installed here without a placement record one, from this node's + * own location rows, once this node knows the address they are filed under. Run + * again until it succeeds. + * @returns {Promise} + */ +async function backfillPlacements() { + await untilSucceeds('Backfilling placement records', async () => { + const localSocketAddr = await fluxNetworkHelper.waitForLocalSocketAddress(); + const installed = await dbHelper.findInDatabase( + dbHelper.databaseConnection().db(config.database.appslocal.database), + localAppsInformation, + {}, + { projection: { _id: 0, name: 1 } }, + ); + await appPlacements.backfillPlacements(installed.map((app) => app.name), localSocketAddr); + }); +} + +module.exports = { backfillPlacements }; diff --git a/ZelBack/src/services/nodeConfirmationService.js b/ZelBack/src/services/nodeConfirmationService.js index af58554dbc..f0de67e3ef 100644 --- a/ZelBack/src/services/nodeConfirmationService.js +++ b/ZelBack/src/services/nodeConfirmationService.js @@ -15,6 +15,7 @@ let daemonStale = false; let messageCapable = false; let started = false; let lastSuccessfulPoll = null; +let capabilityCheck = Promise.resolve(); const confirmedGate = new AsyncGate(); const confirmationStatusGate = new AsyncGate(); @@ -123,6 +124,23 @@ async function poll() { return; } + await checkMessageCapability(); +} + +/** + * Whether this node can be heard: the address benchmark reports is listed under + * this node's key. Told to the capability listeners when it changes. One check + * runs at a time, so a check started by a poll and one started by the node + * learning its address cannot both report the same change. + * @returns {Promise} + */ +function checkMessageCapability() { + capabilityCheck = capabilityCheck.then(readMessageCapability, readMessageCapability); + return capabilityCheck; +} + +async function readMessageCapability() { + const prevMessageCapable = messageCapable; let newMessageCapable = false; try { if (!ourPubkey) { @@ -161,6 +179,15 @@ async function poll() { } } +/** + * The node has an address after having none: whether it can be heard is asked + * at once. + * @returns {void} + */ +function onOwnAddressKnown() { + if (daemonConfirmed) checkMessageCapability().catch((error) => log.error(error)); +} + function scheduleNext() { setTimeout(async () => { await poll(); @@ -171,6 +198,7 @@ function scheduleNext() { async function start() { if (started) return; started = true; + fluxNetworkHelper.onLocalSocketAddressKnown(onOwnAddressKnown); await poll(); confirmationStatusGate.open(); scheduleNext(); diff --git a/ZelBack/src/services/nodeHealthService.js b/ZelBack/src/services/nodeHealthService.js index b333917565..3e7dab9e5c 100644 --- a/ZelBack/src/services/nodeHealthService.js +++ b/ZelBack/src/services/nodeHealthService.js @@ -195,6 +195,13 @@ async function checkNodeFitness() { // that has stopped answering, and daemonHealthMonitor acts on one that stays so. checks.daemon = daemonServiceMiscRpcs.isDaemonSynced().data.synced ? CHECK.OK : CHECK.DEGRADED; + // udp: whether this node's ports answer over UDP, as the availability check + // last established it. Reported, never refused on here: whether a port closed + // over UDP takes the node out of service is that check's to decide. + const udp = fluxNetworkHelper.getUdpReachability(); + if (udp === null) checks.udp = CHECK.UNMEASURED; + else checks.udp = udp.reachable ? CHECK.OK : CHECK.DEGRADED; + // hardware: the node must still meet its tier requirements const hardware = await confirmNodeTierHardware(); if (hardware.result === CHECK.FAIL) { diff --git a/ZelBack/src/services/residentialNodeDosService.js b/ZelBack/src/services/residentialNodeDosService.js index a8080752b3..b119e90233 100644 --- a/ZelBack/src/services/residentialNodeDosService.js +++ b/ZelBack/src/services/residentialNodeDosService.js @@ -161,6 +161,13 @@ let lastDecision = null; // named rather than encoded in a magic origin. let lastEvacuationAt = null; +// A clear of the settle marker that failed: the marker still holds the window a +// verdict that has since flipped accumulated, and nothing may count it. +let settleMarkerUncleared = false; +// A departure whose time could not be written to the marker, written again on +// each tick until it is: the next process reopens the gate early without it. +let unpersistedEvacuationAt = null; + // Whether this node yet knows what it is running. Starts false and is only // raised by the orchestrator's own signal - the same one appSpawner waits on. // globalState.spawnerPaused is NOT this signal: it initialises to false, so @@ -258,7 +265,8 @@ async function isResidential() { } /** - * The settling marker as stored, or null when it cannot be read. + * The settling marker as stored, or null when there is none. Rejects when it + * cannot be read. * * Persisted rather than held in memory: a counter of consecutive evaluations in * memory is reset by restarting FluxOS, which would make restarting on a timer a @@ -266,16 +274,11 @@ async function isResidential() { * @returns {Promise} The marker document, or null. */ async function getSettleMarker() { - try { - const db = dbHelper.databaseConnection(); - if (!db) return null; - const database = db.db(config.database.local.database); - const marker = await dbHelper.findOneInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }); - return marker || null; - } catch (error) { - log.warn(`residentialNodeDos - could not read settle marker: ${error.message}`); - return null; - } + const db = dbHelper.databaseConnection(); + if (!db) throw new Error('no database connection'); + const database = db.db(config.database.local.database); + const marker = await dbHelper.findOneInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }); + return marker || null; } /** @@ -318,10 +321,16 @@ function creditForGap(gapMs) { * the router the way to postpone enforcement. * @param {number} now Epoch ms. * @returns {Promise} Observed milliseconds, or null when the marker - * cannot be written. + * cannot be read or written. A marker that cannot be read is not written. */ async function noteVerdictConfirmed(now) { - const marker = await getSettleMarker(); + let marker; + try { + marker = await getSettleMarker(); + } catch (error) { + log.warn(`residentialNodeDos - could not read settle marker: ${error.message}`); + return null; + } // A marker written before this node kept an observed total has no record of // what was watched, only of when the clock started - and that is exactly the // measure being replaced. It starts accumulating from here rather than being @@ -377,16 +386,23 @@ async function noteVerdictConfirmed(now) { /** * Clear the settling window. Only a verdict flip does this - the node is no - * longer residential, or is now ArcaneOS. + * longer residential, or is now ArcaneOS. A clear that fails is remembered, and + * the window is not counted again until a clear succeeds. + * @returns {Promise} Whether the marker is cleared. */ async function clearSettleMarker() { try { const db = dbHelper.databaseConnection(); - if (!db) return; + if (!db) throw new Error('no database connection'); const database = db.db(config.database.local.database); await dbHelper.findOneAndDeleteInDatabase(database, startupCollection, { _id: SETTLE_MARKER_KEY }, {}); + settleMarkerUncleared = false; + unpersistedEvacuationAt = null; + return true; } catch (error) { + settleMarkerUncleared = true; log.warn(`residentialNodeDos - could not clear settle marker: ${error.message}`); + return false; } } @@ -558,17 +574,20 @@ function noteEvacuated(appName, now = monotonicMs()) { async function persistLastEvacuationAt(now) { try { const db = dbHelper.databaseConnection(); - if (!db) return; + if (!db) throw new Error('no database connection'); const database = db.db(config.database.local.database); await dbHelper.findOneAndUpdateInDatabase( database, startupCollection, { _id: SETTLE_MARKER_KEY }, - { $set: { lastEvacuationAt: now } }, + // never backwards: a retried older time does not replace a newer one + { $max: { lastEvacuationAt: now } }, { upsert: true }, ); + if (unpersistedEvacuationAt === now) unpersistedEvacuationAt = null; } catch (error) { - log.warn(`residentialNodeDos - could not record the departure time: ${error.message}`); + unpersistedEvacuationAt = Math.max(unpersistedEvacuationAt ?? 0, now); + log.warn(`residentialNodeDos - could not record the departure time, written again on the next tick: ${error.message}`); } } @@ -723,6 +742,13 @@ async function runResidentialPolicy(deps) { return true; } + if (settleMarkerUncleared && !await clearSettleMarker()) { + evacuating = false; + log.info('residentialNodeDos - the settle marker of an earlier verdict is not yet cleared, evacuation stays off'); + return false; + } + if (unpersistedEvacuationAt !== null) await persistLastEvacuationAt(unpersistedEvacuationAt); + const now = Date.now(); const observedMs = await noteVerdictConfirmed(now); if (observedMs !== null) observedWindowMs = observedMs; diff --git a/ZelBack/src/services/serviceManager.js b/ZelBack/src/services/serviceManager.js index ab1acdc08b..4967afa496 100644 --- a/ZelBack/src/services/serviceManager.js +++ b/ZelBack/src/services/serviceManager.js @@ -6,10 +6,14 @@ const https = require('https'); const cacheManager = require('./utils/cacheManager').default; const log = require('../lib/log'); const dbHelper = require('./dbHelper'); +const { untilSucceeds } = require('./utils/untilSucceeds'); const explorerService = require('./explorerService'); const fluxCommunication = require('./fluxCommunication'); const networkStateService = require('./networkStateService'); const fluxNetworkHelper = require('./fluxNetworkHelper'); +const databaseHealthService = require('./databaseHealthService'); +const hostStallService = require('./hostStallService'); +const appKill = require('./appLifecycle/appKill'); // App modular services - replacing appsService const appInstaller = require('./appLifecycle/appInstaller'); const appUninstaller = require('./appLifecycle/appUninstaller'); @@ -46,6 +50,7 @@ const appQueryService = require('./appQuery/appQueryService'); const daemonServiceMiscRpcs = require('./daemonService/daemonServiceMiscRpcs'); const daemonServiceUtils = require('./daemonService/daemonServiceUtils'); const paymentPositions = require('./migrations/paymentPositions'); +const placements = require('./migrations/placements'); const fluxService = require('./fluxService'); const geolocationService = require('./geolocationService'); const ipLocationSync = require('./appPlacement/ipLocationSync'); @@ -69,6 +74,7 @@ const uplinkService = require('./uplinkService'); const nodeConfirmationService = require('./nodeConfirmationService'); const appTamperingDetectionService = require('./appTamperingDetectionService'); const appsRuntimeState = require('./appManagement/appsRuntimeState'); +const appPlacements = require('./appManagement/appPlacements'); const imageUpdateService = require('./imageUpdateService'); const { version: fluxVersion } = require('../../../package.json'); // const throughputLogger = require('./utils/throughputLogger'); @@ -99,161 +105,152 @@ const dosState = { testingPort: null, nextTestingPort: null, originalPortFailed: null, - lastUPNPMapFailed: false, + routerRefusalsSeen: 0, }; const portsNotWorking = new Set(); const appsStorageViolations = []; /** - * Remove rows that duplicate a would-be-unique key, keeping the newest of each. - * - * A recovery strategy for ensureIndex: when a unique build fails because the - * collection already holds rows that violate it, this makes the data conform to - * the invariant the index DECLARES - it deletes duplicates on the key the index - * says must be unique, which is enforcing a contract rather than losing data. - * Only safe where the key IS the row's identity, so it is passed in per build by - * the caller that knows the collection, never applied by default. A rollup whose - * duplicates must be summed rather than dropped (the tampering incident count) - * belongs to its owning service instead - see the note on ensureIndex. - * - * Keeps the newest per group (rows sort by _id, which is time-ordered), honours - * the index's partialFilterExpression so it only touches rows the index covers, - * and returns how many it removed. - * - * @param {object} collection - a mongo collection handle - * @param {object} spec - the index key, e.g. { hash: 1 } - * @param {object} options - the index options (read for partialFilterExpression) - * @returns {Promise} rows removed + * Brings every collection boot depends on to its schema: indexes, the merges + * their unique indexes need, and the one-off repairs. Every step is safe to + * repeat, and any failure rejects. + * @returns {Promise} */ -async function dedupeByKey(collection, spec, options = {}) { - const groupId = {}; - Object.keys(spec).forEach((key, i) => { groupId[`k${i}`] = `$${key}`; }); - // Held in memory deliberately, and measured rather than assumed: run against a - // live node's collection unioned with itself until every key appeared 16 times - // - 1,030,208 rows, the duplicate state this exists to repair - it finished in - // under 2s without spilling. The $sort adds nothing on top while it stays an - // index walk on _id, which it is for a spec with no partialFilterExpression; - // the first partial index to use this wants re-measuring, because the $match - // ahead of the sort is what would make the sort blocking. allowDiskUse is not - // set: it would take a mongo below 6.0, where the cap errors instead of - // spilling, and the network floor is moving past that. - // - // ids[0] rather than $max: $group does not document that it carries a - // preceding sort into an accumulator, and that non-guarantee is about results - // merged from several sources - this is one standalone mongod. Checked against - // 64,388 real duplicate groups, ids[0] was the newest in all 64,388. - const pipeline = [ - ...(options.partialFilterExpression ? [{ $match: options.partialFilterExpression }] : []), - { $sort: { _id: -1 } }, - { $group: { _id: groupId, ids: { $push: '$_id' } } }, - { $match: { 'ids.1': { $exists: true } } }, - ]; - const groups = await collection.aggregate(pipeline).toArray(); - const toRemove = groups.flatMap((group) => group.ids.slice(1)); - if (!toRemove.length) return 0; - await collection.deleteMany({ _id: { $in: toRemove } }); - return toRemove.length; -} - -/** - * Assert one index, healing the failures that are recoverable. - * - * - a pre-existing index with conflicting OPTIONS (IndexOptionsConflict / - * IndexKeySpecsConflict) is dropped by its real name and recreated; - * - a unique build blocked by DUPLICATE ROWS runs the caller's `recover` - * strategy (see dedupeByKey) and rebuilds, so the node ends up WITH the - * index rather than running degraded without it; - * - anything else rethrows. - * - * The rethrow is deliberate and is NOT the blanket swallow it replaced. Index - * setup runs before any service or interval starts, so the 15s startFluxFunctions - * retry re-runs it safely: a TRANSIENT failure (mongo mid-election, a slow-disk - * blip) heals on the next pass instead of being skipped until the next reboot, - * and a genuinely UNRECOVERABLE database wedges loudly - which is correct, since - * a node whose DB cannot hold its schema cannot serve apps and appremove would - * not rescue it. The realistic wedge that finding motivated - a unique index - * over rows that already violate it - is repaired above, not hidden. - * - * TRUE NORTH: eventually every collection owns its own schema-prepare - its - * index spec plus whatever dedupe or merge its data needs - the way - * appsRuntimeState.prepareCollection and appTamperingDetectionService already - * do, and boot just invokes those prepare functions. That turns this ~40-call - * imperative block into a set of owned, individually testable units. This - * function is the increment toward it, not the destination; a full move of the - * remaining builds is a separate refactor, out of scope for the PR that added it. - * - * @param {object} collection - a mongo collection handle - * @param {object} spec - the index key - * @param {object} [options] - the index options - * @param {(collection: object, spec: object, options: object) => Promise} [recover] - * run when a unique build is blocked by existing duplicate rows - */ -async function ensureIndex(collection, spec, options = {}, recover = null) { - try { - await collection.createIndex(spec, options); - } catch (err) { - const conflict = err && (err.codeName === 'IndexOptionsConflict' || err.codeName === 'IndexKeySpecsConflict'); - if (conflict) { - const specKeys = JSON.stringify(spec); - const indexes = await collection.listIndexes().toArray(); - const match = indexes.find((idx) => JSON.stringify(idx.key) === specKeys); - if (match?.name) { - log.warn(`ensureIndex - conflicting index '${match.name}' on ${collection.collectionName} (key: ${specKeys}), dropping and recreating`); - await collection.dropIndex(match.name); +async function prepareDatabaseSchema() { + log.info('Preparing local database...'); + // Index builds: they run on the maintenance client. + const db = dbHelper.maintenanceConnection() || dbHelper.databaseConnection(); + const database = db.db(config.database.local.database); + // Logins survive a restart: every request re-verifies its signature and + // re-checks its privilege against the node's current config, and each + // session collection's TTL index expires its rows. A login row never holds + // the session's signature (loginSignatures does, for a minute); one that + // does loses it here. + await dbHelper.updateInDatabase(database, config.database.local.collections.loggedUsers, { signature: { $exists: true } }, { $unset: { signature: '' } }); + // Named literally because they are no longer part of the schema: the payment + // request and receipt collections outlived the endpoint that wrote them, and + // a node's word was never the payment record - the chain is. Dropping is + // idempotent, so this costs one 'ns not found' per boot once they are gone. + await Promise.all(['activepaymentrequests', 'completedpayments'].map((orphan) => dbHelper + .dropCollection(database, orphan).catch((error) => { + if (error.message !== 'ns not found') { + log.error(error); } - await collection.createIndex(spec, options); - return; - } - const duplicate = err && (err.code === 11000 || err.codeName === 'DuplicateKey'); - if (duplicate && recover) { - const removed = await recover(collection, spec, options); - log.warn(`ensureIndex - ${collection.collectionName} (key: ${JSON.stringify(spec)}) held ${removed} row(s) violating a unique index; removed and rebuilding`); - await collection.createIndex(spec, options); - return; - } - throw err; - } -} - -/** - * Assert every index one collection needs, in a single command. - * - * mongo's index build protocol has a fixed cost per BUILD - register, start, - * scan, wait for commit quorum, commit, log - and it does not care that the - * collection is empty. A node asserting its schema one index at a time pays - * that cost 34 times over 14 collections; createIndexes pays it once per - * collection. A node boots faster for it, and the integration harness, where - * ten nodes share one mongod and every database is new, feels it ten times over - * (measured: 938 concurrent builds per ten-node fleet). - * - * The batch is the fast path, not the only one. ensureIndex heals two failures - * that need to be attributed to a single index - an options conflict it drops - * and rebuilds, and a unique build blocked by duplicate rows it repairs through - * the caller's strategy - and a batch rejection does not say which member - * failed. So any error falls back to asserting them one at a time, which is - * exactly the behaviour that existed before this function. - * - * @param {object} collection - a mongo collection handle - * @param {Array} specs - `{ key, ...indexOptions, recover }` per index, - * where `recover` is ours and never reaches mongo - */ -async function ensureIndexes(collection, specs) { - try { - await collection.createIndexes(specs.map((spec) => { - // `recover` is a FluxOS concern; mongo is handed the index model alone - const model = { ...spec }; - delete model.recover; - return model; - })); - return; - } catch (error) { - log.warn(`ensureIndexes - batch of ${specs.length} on ${collection.collectionName} failed (${error.codeName || error.message}); asserting one at a time`); - } - // eslint-disable-next-line no-restricted-syntax - for (const { key, recover = null, ...options } of specs) { - // eslint-disable-next-line no-await-in-loop - await ensureIndex(collection, key, options, recover); - } + }))); + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ + { key: { createdAt: 1 }, expireAfterSeconds: 14 * 24 * 60 * 60 }, + ]); + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ + { key: { createdAt: 1 }, expireAfterSeconds: 900 }, + ]); + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ + { key: { createdAt: 1 }, expireAfterSeconds: 900 }, + ]); + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.loginSignatures), [ + { key: { createdAt: 1 }, expireAfterSeconds: 60 }, + { key: { loginPhrase: 1 } }, + ]); + // legacy pre-incident-schema rows expire via detectedAt; current incident + // documents expire via lastSeen. The tamper service purges pre-schema + // rows at startup, so the detectedAt pair only matters where old code + // still writes; drop it once the fleet is past the incident schema. + await dbHelper.ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ + { key: { detectedAt: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days + { key: { appName: 1, detectedAt: -1 }, name: 'appName_detectedAt' }, + { key: { lastSeen: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days + { key: { appName: 1, eventType: 1, lastSeen: -1 }, name: 'appName_eventType_lastSeen' }, + ]); + // The unique incident-rollup index lives with its owner: duplicate rollups + // must have their counts SUMMED, not one dropped, so the merge needs the + // collection's own knowledge rather than a generic dedupe. See + // prepareIncidentRollup. + await appTamperingDetectionService.prepareIncidentRollup(); + await appTamperingDetectionService.checkNodeReboot(); + // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, + // then enforce one doc per component identifier + await appsRuntimeState.prepareCollection(); + // appPlacements (localzelapps): one record per app + await appPlacements.prepareCollection(); + // The explorer's collections: their indexes are the explorer's to declare. + await explorerService.prepareCollections(); + log.info('Local database prepared'); + log.info('Preparing temporary database...'); + // no need to drop temporary messages + const databaseTemp = db.db(config.database.appsglobal.database); + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ + { key: { receivedAt: 1 }, expireAfterSeconds: tempMsgTtlS }, + ]); + log.info('Temporary database prepared'); + log.info('Preparing Flux Apps locations'); + + // ToDo: Fix all these broken database drops / index creations / removals all over the place. The prior dropIndex was removing the + // index entirely so there was no index at all! + + // The below index is created in the Explorer Service. We need to remove all the database indexing from the Explorer Service. + // It's not the explorer service's responsibility, and other services need these indexes before Explorer Service creates them. + + // It should be the dbService's responsibility that the db is in a state fit for use. + + // we have to create this index again here, as we need it to repair the db. As we were deleting this on every reboot (and it was only created when scannedHeight was 0) + // Creating an index that already exists is a no-op + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ + { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dbHelper.dedupeByKey }, + { key: { txid: 1 }, name: 'query for getting zelapp message based on txid' }, + { key: { height: 1 }, name: 'query for getting zelapp message based on height' }, + { key: { 'appSpecifications.name': 1 }, name: 'query for getting app message based on zelapp specs name' }, + { key: { 'appSpecifications.owner': 1 }, name: 'query for getting app message based on zelapp specs owner' }, + { key: { 'appSpecifications.repotag': 1 }, name: 'query for getting app message based on image' }, + { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, + { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, + ]); + await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); + // TTL is driven by expireAt (set per-document by store functions). + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, + { key: { ip: 1, name: 1 } }, + ]); + log.info('Flux Apps locations prepared'); + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dbHelper.dedupeByKey }, + { key: { broadcastedAt: 1 } }, + { key: { createdAt: 1 } }, + // the events of one type still standing, every move among them + { key: { type: 1, expireAt: 1 }, name: 'events by type and expiry' }, + // the moves that lead to an address + { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, + ]); + log.info('App state events collection prepared'); + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { broadcastedAt: 1 } }, + { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dbHelper.dedupeByKey }, + ]); + log.info('Signed appinstalling broadcasts collection prepared'); + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, + { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, + ]); + log.info('Flux Apps installing locations prepared'); + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, + { key: { name: 1, hash: 1 }, name: 'query for getting flux app install errors location based on specs name and hash' }, + { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, + ]); + log.info('App installing errors locations prepared'); + await dbHelper.ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { broadcastedAt: 1 } }, + { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dbHelper.dedupeByKey }, + ]); + log.info('Signed app installing errors broadcasts collection prepared'); + + // This fixes an issue where the appsMessage db has NaN for valueSat. Once db is repaired on all nodes, + // we can remove this. + await dbHelper.repairNanInAppsMessagesDb(); } /** @@ -273,13 +270,26 @@ async function startFluxFunctions() { // below-floor node stays up holding a sticky DOS, so /flux/info names the // version found and the version required. fluxNetworkHelper.checkNodeJsVersionAllowed(); + // Before the dependencies are waited for: a stalled one is what this sees. + // Awaited so a trip recorded in this boot holds the node before boot acts. + await hostStallService.start(); // Hard dependencies — nothing starts until these are confirmed. await dbHelper.waitForMongo(); + databaseHealthService.watch(dbHelper.databaseConnection(), 'main'); + databaseHealthService.watch(dbHelper.maintenanceConnection(), 'maintenance'); + // As soon as MongoDB answers: a below-floor node stays up holding a sticky + // DOS, so /flux/info names the version found and the version required. + fluxNetworkHelper.checkMongoDbVersionAllowed(await dbHelper.getMongoDbVersion()); await dockerService.waitForDocker(); + const dockerInfo = await dockerService.dockerInfo().catch(() => null); + // Where docker keeps its records, read while it answers, for a kill made + // while it does not. + if (dockerInfo?.DockerRootDir) appKill.usePaths({ dockerRoot: dockerInfo.DockerRootDir }); // As soon as Docker answers, before the container firewall is applied: a // below-floor node stays up holding a sticky DOS, so /flux/info names the // version found and the version required. fluxNetworkHelper.checkDockerVersionAllowed(await dockerService.dockerVersion().then((version) => version.Version, () => null)); + await fluxNetworkHelper.checkCgroupsAllowed(dockerInfo?.CgroupDriver ?? null); // Before any work that a restart would cut short: a registration that // needs its kill timeout raised restarts FluxOS, and this waits for it. await fluxService.ensurePm2KillTimeout(); @@ -322,138 +332,8 @@ async function startFluxFunctions() { log.info('System service initiated'); fluxadmService.start(); log.info('Fluxadm access service initiated'); - log.info('Preparing local database...'); + await untilSucceeds('Database schema preparation', prepareDatabaseSchema); const db = dbHelper.databaseConnection(); - const database = db.db(config.database.local.database); - // Logins survive a restart: every request re-verifies its signature and - // re-checks its privilege against the node's current config, and each - // session collection's TTL index expires its rows. A login row never holds - // the session's signature (loginSignatures does, for a minute); one that - // does loses it here. - await dbHelper.updateInDatabase(database, config.database.local.collections.loggedUsers, { signature: { $exists: true } }, { $unset: { signature: '' } }).catch((error) => { - log.error(error); - }); - // Named literally because they are no longer part of the schema: the payment - // request and receipt collections outlived the endpoint that wrote them, and - // a node's word was never the payment record - the chain is. Dropping is - // idempotent, so this costs one 'ns not found' per boot once they are gone. - await Promise.all(['activepaymentrequests', 'completedpayments'].map((orphan) => dbHelper - .dropCollection(database, orphan).catch((error) => { - if (error.message !== 'ns not found') { - log.error(error); - } - }))); - await ensureIndexes(database.collection(config.database.local.collections.loggedUsers), [ - { key: { createdAt: 1 }, expireAfterSeconds: 14 * 24 * 60 * 60 }, - ]); - await ensureIndexes(database.collection(config.database.local.collections.activeLoginPhrases), [ - { key: { createdAt: 1 }, expireAfterSeconds: 900 }, - ]); - await ensureIndexes(database.collection(config.database.local.collections.activeSignatures), [ - { key: { createdAt: 1 }, expireAfterSeconds: 900 }, - ]); - await ensureIndexes(database.collection(config.database.local.collections.loginSignatures), [ - { key: { createdAt: 1 }, expireAfterSeconds: 60 }, - { key: { loginPhrase: 1 } }, - ]); - // legacy pre-incident-schema rows expire via detectedAt; current incident - // documents expire via lastSeen. The tamper service purges pre-schema - // rows at startup, so the detectedAt pair only matters where old code - // still writes; drop it once the fleet is past the incident schema. - await ensureIndexes(database.collection(config.database.local.collections.appTamperingEvents), [ - { key: { detectedAt: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'detectedAt_ttl' }, // 30 days - { key: { appName: 1, detectedAt: -1 }, name: 'appName_detectedAt' }, - { key: { lastSeen: 1 }, expireAfterSeconds: 30 * 24 * 60 * 60, name: 'lastSeen_ttl' }, // 30 days - { key: { appName: 1, eventType: 1, lastSeen: -1 }, name: 'appName_eventType_lastSeen' }, - ]); - // The unique incident-rollup index lives with its owner: duplicate rollups - // must have their counts SUMMED, not one dropped, so the merge needs the - // collection's own knowledge rather than a generic dedupe. See - // prepareIncidentRollup. - await appTamperingDetectionService.prepareIncidentRollup(); - await appTamperingDetectionService.checkNodeReboot(); - // appsRuntimeState (localzelapps): merge any pre-unique-index duplicate docs, - // then enforce one doc per component identifier - await appsRuntimeState.prepareCollection(); - log.info('Local database prepared'); - log.info('Preparing temporary database...'); - // no need to drop temporary messages - const databaseTemp = db.db(config.database.appsglobal.database); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsTemporaryMessages), [ - { key: { receivedAt: 1 }, expireAfterSeconds: tempMsgTtlS }, - ]); - log.info('Temporary database prepared'); - log.info('Preparing Flux Apps locations'); - - // ToDo: Fix all these broken database drops / index creations / removals all over the place. The prior dropIndex was removing the - // index entirely so there was no index at all! - - // The below index is created in the Explorer Service. We need to remove all the database indexing from the Explorer Service. - // It's not the explorer service's responsibility, and other services need these indexes before Explorer Service creates them. - - // It should be the dbService's responsibility that the db is in a state fit for use. - - // we have to create this index again here, as we need it to repair the db. As we were deleting this on every reboot (and it was only created when scannedHeight was 0) - // Creating an index that already exists is a no-op - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsMessages), [ - { key: { hash: 1 }, name: 'query for getting zelapp message based on hash', unique: true, recover: dedupeByKey }, - { key: { 'appSpecifications.version': 1 }, name: 'query for getting app message based on version' }, - { key: { 'appSpecifications.nodes': 1 }, name: 'query for getting app message based on nodes' }, - ]); - await dbHelper.ensureNewestAppMessageIndex(databaseTemp.collection(config.database.appsglobal.collections.appsMessages)); - // TTL is driven by expireAt (set per-document by store functions). Migrate from old broadcastedAt-based TTL. - await databaseTemp.collection(config.database.appsglobal.collections.appsLocations).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsLocations), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'query for getting zelapp location based on zelapp specs name' }, - { key: { ip: 1, name: 1 } }, - ]); - log.info('Flux Apps locations prepared'); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appStateEvents), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { ip: 1, type: 1, dedupKey: 1 }, unique: true, recover: dedupeByKey }, - { key: { broadcastedAt: 1 } }, - { key: { createdAt: 1 } }, - // the events of one type still standing, every move among them - { key: { type: 1, expireAt: 1 }, name: 'events by type and expiry' }, - // the moves that lead to an address - { key: { 'data.newIP': 1 }, name: 'moves by the address they lead to', partialFilterExpression: { type: 'ipchanged' } }, - ]); - log.info('App state events collection prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingBroadcasts), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { broadcastedAt: 1 } }, - { key: { 'data.name': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, - ]); - log.info('Signed appinstalling broadcasts collection prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingLocations), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'query for getting flux app install location based on specs name' }, - { key: { name: 1, ip: 1 }, name: 'query for getting flux app install location based on specs name and node ip' }, - ]); - log.info('Flux Apps installing locations prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).dropIndex('cachedAt_1').catch(() => {}); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsLocations), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'query for getting flux app install errors location based on specs name' }, - { key: { name: 1, hash: 1 }, name: 'query for getting flux app install errors location based on specs name and hash' }, - { key: { name: 1, hash: 1, ip: 1 }, name: 'query for getting flux app install errors location based on specs name and hash and node ip' }, - ]); - log.info('App installing errors locations prepared'); - await databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts).dropIndex('broadcastedAt_1').catch(() => {}); - await ensureIndexes(databaseTemp.collection(config.database.appsglobal.collections.appsInstallingErrorsBroadcasts), [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { broadcastedAt: 1 } }, - { key: { 'data.name': 1, 'data.hash': 1, 'data.ip': 1 }, unique: true, recover: dedupeByKey }, - ]); - log.info('Signed app installing errors broadcasts collection prepared'); - - // This fixes an issue where the appsMessage db has NaN for valueSat. Once db is repaired on all nodes, - // we can remove this. - await dbHelper.repairNanInAppsMessagesDb(); // The location table this node already holds, brought back as soon as the // database is up. Detached and best-effort - every consumer degrades safely @@ -514,6 +394,9 @@ async function startFluxFunctions() { appStartupManager.manageAppsOnBoot(bootContext).catch((error) => { log.error(`App startup manager error: ${error.message}`); }); + // Not awaited: it waits for this node's address, and the announcer, the one + // reader of what it writes, waits for it. + placements.backfillPlacements(); // Above the daemon wait, because syncthing has nothing to do with the // daemon: the sentinel talks to syncthing's own API and reads userconfig, @@ -539,7 +422,7 @@ async function startFluxFunctions() { log.error('fluxd is running without addressindex=1. FluxOS needs the address index to order app messages. Set addressindex=1 in flux.conf and reindex the daemon, or reinstall the node. FluxOS is stopping.'); process.exit(1); } - await paymentPositions.backfillPaymentPositions(); + await untilSucceeds('Payment position backfill', () => paymentPositions.backfillPaymentPositions()); // Initialize app sync orchestrator and spawner const orchestrator = new AppSyncOrchestrator({ @@ -872,9 +755,7 @@ async function startFluxFunctions() { ); }, bootDelay(3 * 60 * 1000)); nodeStatusMonitor.initialize(appQueryService.installedApps, appUninstaller.removeAppLocally); - setTimeout(() => { - nodeStatusMonitor.monitorNodeStatus(appQueryService.installedApps, appUninstaller.removeAppLocally); - }, bootDelay(1.5 * 60 * 1000)); + nodeStatusMonitor.start(appQueryService.installedApps, appUninstaller.removeAppLocally); // Start the syncthing/masterSlave deciders once boot container state has settled // (the same AsyncGate the reconciler starts on), not after a fixed delay. Each // decider self-gates per cycle on its own prerequisites (mounts, syncthing health, @@ -894,10 +775,10 @@ async function startFluxFunctions() { // concurrently rather than after a timed offset. // The election reads the busy lists and the receive-only cache off // globalState itself at each decision; they are not parameters. The - // getters return snapshots and masterSlaveApps re-invokes itself forever, - // so anything captured at this call is frozen at boot and goes quietly - // stale - which is exactly how the backup/restore guard once died. - advancedWorkflows.masterSlaveApps( + // getters return snapshots and the election runs for the life of the + // process, so anything captured at this call is frozen at boot and goes + // quietly stale. + advancedWorkflows.startMasterSlaveApps( globalState, appQueryService.installedApps, appQueryService.listRunningApps, @@ -958,9 +839,7 @@ async function startFluxFunctions() { module.exports = { startFluxFunctions, - ensureIndex, - ensureIndexes, - dedupeByKey, + prepareDatabaseSchema, // testing exports dosState, }; diff --git a/ZelBack/src/services/syncthingService.js b/ZelBack/src/services/syncthingService.js index 0780f52908..0e81be9fba 100644 --- a/ZelBack/src/services/syncthingService.js +++ b/ZelBack/src/services/syncthingService.js @@ -876,6 +876,17 @@ async function getDbLocalChanged(folder, page = 1, perpage = LOCAL_CHANGED_PAGE_ return request('get', `/rest/db/localchanged?folder=${folder}&page=${page}&perpage=${perpage}`); } +/** + * One item of a folder: this node's record of it (`local`), the cluster's version + * (`global`), and the connected devices that hold that version (`availability`). + * @param {string} folder Folder ID. + * @param {string} file The item's path within the folder. + * @returns {Promise<{local: object, global: object, availability: Array}>} + */ +async function getDbFile(folder, file) { + return request('get', `/rest/db/file?${qs.stringify({ folder, file })}`); +} + /** * A folder's local changes, handed to a caller one page at a time. * @@ -2737,6 +2748,7 @@ module.exports = { nonFileEntryBytes, nonFileEntryBytesForVersion, statusInFileBytes, + getDbFile, getDbLocalChanged, eachDbLocalChanged, dbRevert, diff --git a/ZelBack/src/services/upnpService.js b/ZelBack/src/services/upnpService.js index 2650219861..865f94984f 100644 --- a/ZelBack/src/services/upnpService.js +++ b/ZelBack/src/services/upnpService.js @@ -1,3 +1,4 @@ +const { EventEmitter } = require('node:events'); const config = require('config'); const natUpnp = require('@runonflux/nat-upnp'); const serviceHelper = require('./serviceHelper'); @@ -10,14 +11,22 @@ const ufw = require('./utils/ufw'); const fluxadmPort = require('./fluxadmPort'); const { Privilege, authOf } = require('./utils/privileges'); -const client = new natUpnp.Client(); +// config.upnp.gatewayUrl names the router's device description, used in place +// of discovering the router; the node's address is then config.upnp.nodeIp. +// A discovered router is kept between calls, and found again when it stops +// answering where it was. +const client = config.upnp.gatewayUrl + ? new natUpnp.Client({ url: config.upnp.gatewayUrl, localAddress: config.upnp.nodeIp || '127.0.0.1' }) + : new natUpnp.Client({ cacheGateway: true }); const FLUXADM_MAPPING_DESCRIPTION = 'Flux_Fluxadm_SSH'; // The descriptions this code gives the mappings it makes. Only these are ever // cleaned up: 'Flux_manual_entry' is the operator's own request through the // API, and anything else on the router - the owner's own forwards, another -// program's, a description the router truncated - is not ours to judge. +// program's, a description the router truncated - is not ours to judge. A +// description is compared as routers store it (natUpnp.canonicalDescription): +// FRITZ!Box keeps 'Flux_App_web' as 'Flux.App.web'. const OWNED_MAPPING_DESCRIPTIONS = new Set([ 'Flux_Backend_API', 'Flux_Backend_API_SSL', @@ -32,6 +41,63 @@ const OWNED_MAPPING_PREFIXES = ['Flux_App_', 'Flux_Prelaunch_App_']; // port mapped here is in use - a test port mid-test, an app installing - // whatever the app table says so far. const mappedHere = new Set(); +// The protocols of each port whose latest map here created the mapping or found +// it already this node's, by port: what removeMapUpnpPort removes. A router +// refuses to map a port another machine holds, so a mapping it took is this +// node's, and one it refused never is. +const mappedProtocols = new Map(); + +// Removals of this node's own mappings the router refused, by `port/protocol`: +// each is asked again, one at a time, after config.upnp.removalRetryFirstMs, +// the wait doubling to config.upnp.removalRetryMaxMs, until the router removes +// it or holds no such entry. A port this node maps again is in use, and its +// pending removal is dropped. +const pendingRemovals = new Map(); +let pendingRemovalTurn = Promise.resolve(); + +// Emits 'removalPending' when a removal becomes pending with none pending +// before, 'removalsClear' when the last pending removal goes, and +// 'mappingRefused' each time a refusal is counted in routerRefusals. +const routerEvents = new EventEmitter(); + +/** + * Makes a removal pending under `key`, in place of any pending there. + * @param {string} key `port/protocol` + * @param {object} pending + */ +function setPendingRemoval(key, pending) { + const wasClear = pendingRemovals.size === 0; + clearTimeout(pendingRemovals.get(key)?.timer); + pendingRemovals.set(key, pending); + if (wasClear) routerEvents.emit('removalPending'); +} + +/** + * Ends the pending removal under `key`, if there is one. + * @param {string} key `port/protocol` + */ +function deletePendingRemoval(key) { + const pending = pendingRemovals.get(key); + if (!pending) return; + clearTimeout(pending.timer); + pendingRemovals.delete(key); + if (pendingRemovals.size === 0) routerEvents.emit('removalsClear'); +} + +/** + * Records whether this node's latest map of one protocol of a port left the + * router mapping it here. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @param {boolean} mapped + */ +function recordMapped(port, protocol, mapped) { + const protocols = mappedProtocols.get(port) || new Set(); + if (mapped) protocols.add(protocol); + else protocols.delete(protocol); + if (protocols.size) mappedProtocols.set(port, protocols); + else mappedProtocols.delete(port); +} // The cleanup pass runs in a wall-clock slot of its own, laid out as #1710 laid // out its refresh: a 10-minute cycle of eight 75-second slots, the slot chosen // by the api port's tens digit. Nodes behind one public address have distinct @@ -40,50 +106,122 @@ const mappedHere = new Set(); const CLEANUP_CYCLE_MS = 10 * 60 * 1000; const CLEANUP_SLOT_COUNT = 8; const CLEANUP_SLOT_MS = CLEANUP_CYCLE_MS / CLEANUP_SLOT_COUNT; -// The library's requests have no timeout of their own: a router that restarts -// with one in flight never answers it. -const CLEANUP_REQUEST_TIMEOUT_MS = 15 * 1000; -// The listing stops asking this long before the slot ends, so that what it has -// listed by then still has time to be removed. +// The listing has to finish this long before the slot ends, so that what it +// lists still has time to be removed. const CLEANUP_REMOVAL_RESERVE_MS = 15 * 1000; +// A node verifying UPnP on a full table lists it at once, without waiting for +// its slot: it cannot run until it has verified. It waits its slot's number +// times this first, so siblings starting together - a power cut, a router +// restart, one update reaching them all - take turns: a router serves one +// request at a time, and six listings at once took 23 to 38 s each where one +// alone took 0.2 s. +const BOOT_LISTING_STAGGER_MS = 5 * 1000; const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); +const NO_SUCH_ENTRY = 714; + +// The refusals of a mapping that name the port or the request as the problem: +// invalid arguments (402), a wildcard or a pair of ports the router will not +// take (716, 724, 726, 727), a lease it will not grant (725), a port another +// machine or another mechanism holds (718, 729). Every other refusal is the +// router's own. Routers do not agree on how they refuse a mapping when their +// table is full: some answer 728 NoPortMapsAvailable, as the standard has it, +// others a generic 501 Action Failed, which some also give for every refresh +// of an entry they hold. A full table cannot be told by its code, so a refusal +// that is not the port's is taken as the router's - unless the router, asked +// after refusing, holds the port for something else: that names the port +// whatever the code, 501 and 606 included. Only 728 says the table is full in +// so many words, and counts as the router's even then. +const PORT_REFUSALS = new Set([402, 716, 718, 724, 725, 726, 727, 729]); +const NO_PORT_MAPS_AVAILABLE = 728; + +// What a test leaves on the router while it runs and removes before it ends. +const TEST_MAPPING_DESCRIPTIONS = new Set(['Flux_UPNP_Mapping_Test', 'Flux_Test_App']); +const TEST_MAPPING_PREFIX = 'Flux_Prelaunch_App_'; + +const MAPPED = Object.freeze({ + mapped: true, code: null, reason: null, routerWide: false, +}); + +let routerRefusalCount = 0; + +/** + * How many of this node's mappings the router has refused since FluxOS started + * for a reason that is its own and not the port's: no answer, or any refusal + * but those PORT_REFUSALS names for a port it does not hold for something + * else. verifyUPNPsupport's test mapping is not counted. Only ever goes up. + * @returns {number} + */ +function routerRefusals() { + return routerRefusalCount; +} + /** - * Settles with `work`, or rejects once `timeoutMs` has passed, with an error - * marked `timedOut`. Only the wait ends: the request is still in flight. - * @param {Promise<*>} work - * @param {number} timeoutMs - * @param {string} what what is waited for, for the error - * @returns {Promise<*>} + * Whether the router maps `port` to this node, port for port, enabled. + * @param {object|null} mapping the router's entry for the port, as client.getMapping reads it + * @param {number} port + * @returns {boolean} */ -async function withinTime(work, timeoutMs, what) { - let timer; - try { - return await Promise.race([ - work, - new Promise((_, reject) => { - timer = setTimeout(() => { - const error = new Error(`${what}: the router did not answer within ${timeoutMs / 1000}s`); - error.timedOut = true; - reject(error); - }, timeoutMs); - }), - ]); - } finally { - clearTimeout(timer); - } +function mapsHere(mapping, port) { + return mapping !== null && mapping.local === true && mapping.enabled && mapping.private.port === port; } -if (config.upnp.gatewayUrl) { - // eslint-disable-next-line global-require - const { Device } = require('@runonflux/nat-upnp/build/src/nat-upnp/device'); - const { gatewayUrl } = config.upnp; - const nodeIp = config.upnp.nodeIp || '127.0.0.1'; - client.getGateway = async () => ({ - gateway: new Device(gatewayUrl), - address: nodeIp, - }); +/** + * Maps one protocol of a port to this node, port for port, with no lease + * (some routers force a short one instead; the mappings are refreshed every + * restore). A refusal is no failure when the router already maps the port + * here: some routers refuse to add an entry they hold, and a full table + * refuses everything, the refresh of a working mapping included. A router + * that does not answer is not asked again. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @param {string} description + * @param {object} [options] + * @param {boolean} [options.counted] whether a refusal of the router's own counts in + * routerRefusals; false for verifyUPNPsupport's test mapping, a probe it + * resolves itself + * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} + * `code` is the router's UPnP error code, null when it gave none; `routerWide` + * whether the refusal is the router's own rather than the port's. + */ +async function mapProtocol(port, protocol, description, { counted = true } = {}) { + if (!Number.isInteger(port) || port < 1 || port > 65535) { + return { + mapped: false, code: null, reason: `${protocol} ${port} not mapped: not a port`, routerWide: false, + }; + } + try { + await client.createMapping({ + public: port, + private: port, + ttl: 0, + protocol, + description, + }); + return MAPPED; + } catch (error) { + const answered = error instanceof natUpnp.UpnpError; + let heldElsewhere = false; + if (answered) { + const held = await client.getMapping({ public: port, protocol }).catch(() => null); + if (mapsHere(held, port)) { + log.info(`UPnP - ${protocol} ${port} refused (${error.message}), but the router already maps it to this node`); + return MAPPED; + } + heldElsewhere = held !== null; + } + const code = answered ? error.code : null; + // no answer, with no code, is the router's too + const routerWide = !PORT_REFUSALS.has(code) && !(heldElsewhere && code !== NO_PORT_MAPS_AVAILABLE); + if (routerWide && counted) { + routerRefusalCount += 1; + routerEvents.emit('mappingRefused'); + } + return { + mapped: false, code, reason: `${protocol} ${port} not mapped: ${error.message}`, routerWide, + }; + } } let upnpMachine = false; @@ -165,44 +303,55 @@ async function verifyUPNPsupport(apiport = config.server.apiport) { // the remove would fail the verify, and a node with a custom api port or a // router IP exits on that. A test that fails leaves it held. mappedHere.add(+apiport + 3); - try { - await client.createMapping({ - public: +apiport + 3, - private: +apiport + 3, - ttl: 0, - description: 'Flux_UPNP_Mapping_Test', - }); - - await serviceHelper.delay(500); - } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed Create Mapping'); - upnpMachine = false; - return false; + let testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test', { counted: false }); + // refused for a reason of the router's own, as a full table is: until UPnP is + // verified the stale mapping cleanup never runs, so the test mappings earlier + // runs left are removed here to make room. A router that did not answer is + // not listed. + if (testMapping.routerWide && testMapping.code !== null) { + let removed = 0; + await serviceHelper.delay(cleanupSlotOf(apiport) * BOOT_LISTING_STAGGER_MS); + try { + removed = await removeLeftoverTestMappings(); + } catch (error) { + log.warn(`VerifyUPNPsupport - left-over test mappings not removed: ${error.message}`); + } + if (removed > 0) testMapping = await mapProtocol(+apiport + 3, 'TCP', 'Flux_UPNP_Mapping_Test', { counted: false }); + // A full table refuses every new mapping, so a node whose router already + // maps its api port to it runs on what it holds: every mapping refused from + // here on counts toward the router DOS, which takes the node out of service + // until a slot frees. + if (!testMapping.mapped) { + const apiMapping = await client.getMapping({ public: +apiport, protocol: 'TCP' }).catch(() => null); + if (mapsHere(apiMapping, +apiport)) { + log.warn(`VerifyUPNPsupport - ${testMapping.reason}, but the router maps this node's api port to it`); + mappedHere.delete(+apiport + 3); + upnpMachine = true; + return true; + } + } } - try { - await client.getMappings(); - - await serviceHelper.delay(500); - } catch (error) { - log.error(error); - log.error('VerifyUPNPsupport - Failed get Mappings'); + if (!testMapping.mapped) { + log.error(`VerifyUPNPsupport - Failed Create Mapping: ${testMapping.reason}`); upnpMachine = false; return false; } + recordMapped(+apiport + 3, 'TCP', true); + await serviceHelper.delay(500); try { - await client.removeMapping({ - public: +apiport + 3, - }); + const mapping = await client.getMapping({ public: +apiport + 3, protocol: 'TCP' }); + if (!mapping) throw new Error(`the router holds no mapping for port ${+apiport + 3}`); await serviceHelper.delay(500); } catch (error) { log.error(error); - log.error('VerifyUPNPsupport - Failed Remove Mapping'); + log.error('VerifyUPNPsupport - Failed get Mapping'); upnpMachine = false; return false; } - mappedHere.delete(+apiport + 3); + // UPnP works once the router has taken the mapping and shows it: a removal it + // refuses is asked again until it goes, and fails nothing here. + await removeMapUpnpPort(+apiport + 3, 'Flux_UPNP_Mapping_Test'); upnpMachine = true; return true; @@ -235,10 +384,10 @@ async function reconcileFluxadmMapping(apiport) { }); return; } - const mappings = await client.getMappings(); - const ours = mappings.some((mapping) => mapping.local - && mapping.public.port === port - && mapping.description === FLUXADM_MAPPING_DESCRIPTION); + const mapping = await client.getMapping({ public: port, protocol: 'TCP' }); + const ours = mapping !== null + && mapping.local === true + && sameDescription(mapping.description, FLUXADM_MAPPING_DESCRIPTION); if (ours) { await client.removeMapping({ public: port, protocol: 'TCP' }); log.info(`fluxadm access - UPnP mapping for port ${port} removed`); @@ -255,41 +404,23 @@ async function reconcileFluxadmMapping(apiport) { */ async function setupUPNP(apiport = config.server.apiport) { try { - await client.createMapping({ - public: +apiport, - private: +apiport, - ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes - description: 'Flux_Backend_API', - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: +apiport + 1, - private: +apiport + 1, - ttl: 0, // Some routers force low ttl if 0, indefinite/default is used. Flux refreshes this every 6 blocks ~ 12 minutes - description: 'Flux_Backend_API_SSL', - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: +apiport - 1, - private: +apiport - 1, - ttl: 0, - description: 'Flux_Home_UI', - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: +apiport + 2, - private: +apiport + 2, - ttl: 0, - description: 'Flux_Syncthing', - }); - - await serviceHelper.delay(500); + const corePortMappings = [ + [+apiport, 'Flux_Backend_API'], + [+apiport + 1, 'Flux_Backend_API_SSL'], + [+apiport - 1, 'Flux_Home_UI'], + [+apiport + 2, 'Flux_Syncthing'], + ]; + // eslint-disable-next-line no-restricted-syntax + for (const [port, description] of corePortMappings) { + // eslint-disable-next-line no-await-in-loop + const result = await mapProtocol(port, 'TCP', description); + if (!result.mapped) { + log.error(`setupUPNP - ${result.reason}`); + return false; + } + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(500); + } await reconcileFluxadmMapping(apiport); @@ -301,71 +432,166 @@ async function setupUPNP(apiport = config.server.apiport) { } /** - * To create mappings for UPnP (Universal Plug and Play) port. + * Maps a port to this node over each of `protocols` in turn, TCP and then UDP + * unless given. + * + * On a failure the port is no longer held here; a protocol the router took is + * still recorded, for removeMapUpnpPort to remove. * @param {number} port Port number. * @param {string} description Port description. - * @returns {Promise} True if port mappings can be created for both TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) protocols. Otherwise false. + * @param {{protocols?: string[]}} [options] + * @returns {Promise<{mapped: boolean, code: number|null, reason: string|null, routerWide: boolean}>} + * the first refusal, as mapProtocol gives it; mapped when every protocol is */ -async function mapUpnpPort(port, description) { +async function mapUpnpPort(port, description, { protocols = ['TCP', 'UDP'] } = {}) { // held from before the router has it: the cleanup pass may list it the // moment it is there mappedHere.add(+port); - try { - await client.createMapping({ - public: port, - private: port, - ttl: 0, - protocol: 'TCP', - description, - }); - - await serviceHelper.delay(500); - - await client.createMapping({ - public: port, - private: port, - ttl: 0, - protocol: 'UDP', - description, - }); - - await serviceHelper.delay(500); + dropPendingRemovals(+port); + let result = MAPPED; + // eslint-disable-next-line no-restricted-syntax + for (const [i, protocol] of protocols.entries()) { + // eslint-disable-next-line no-await-in-loop + if (i > 0) await serviceHelper.delay(500); + // eslint-disable-next-line no-await-in-loop + result = await mapProtocol(+port, protocol, description); + recordMapped(+port, protocol, result.mapped); + if (!result.mapped) break; + } + if (!result.mapped) { + log.error(`UPnP - ${result.reason}`); + mappedHere.delete(+port); + return result; + } + await serviceHelper.delay(500); + return result; +} +/** + * Removes the router's mapping of one protocol of a port. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @returns {Promise} whether the router no longer holds it + */ +async function removeProtocol(port, protocol) { + try { + await client.removeMapping({ public: port, protocol }); return true; } catch (error) { - log.error(error); + if (error instanceof natUpnp.UpnpError && error.code === NO_SUCH_ENTRY) return true; + log.error(`UPnP - ${protocol} ${port} not unmapped: ${error.message}`); return false; } } /** - * To remove TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) port mappings from UPnP (Universal Plug and Play) port. - * @param {number} port Port number. - * @returns {Promise} True if port mappings have been removed for both TCP (Transmission Control Protocol) and UDP (User Datagram Protocol) protocols. Otherwise false. + * Drops the pending removals of a port. + * @param {number} port */ -async function removeMapUpnpPort(port) { - // given up even if the router refuses: what is left is the cleanup pass's - mappedHere.delete(+port); +function dropPendingRemovals(port) { + ['TCP', 'UDP'].forEach((protocol) => deletePendingRemoval(`${port}/${protocol}`)); +} + +/** + * Whether the router still maps one protocol of a port to this node. + * @param {number} port + * @param {string} protocol 'TCP' or 'UDP' + * @returns {Promise} null when the router did not say: a lookup + * it answered with a fault, or did not answer + */ +async function routerMapsHere(port, protocol) { try { - await client.removeMapping({ - public: port, - protocol: 'TCP', - }); + return mapsHere(await client.getMapping({ public: port, protocol }), port); + } catch (error) { + log.error(`UPnP - ${protocol} ${port} not looked up: ${error.message}`); + return null; + } +} - await serviceHelper.delay(500); +/** + * Asks the router once more to remove a pending removal's mapping, in turn + * behind any other pending removal, and schedules the next ask if it refuses + * again. The router is asked first whether the mapping is still this node's: + * one it no longer holds, or holds for something else, ends the removal with + * nothing sent. A lookup it does not settle leaves the mapping counted as + * this node's, and the removal is sent. + * @param {object} pending + */ +function askPendingRemoval(pending) { + pendingRemovalTurn = pendingRemovalTurn.then(async () => { + const key = `${pending.port}/${pending.protocol}`; + if (pendingRemovals.get(key) !== pending) return; + const held = await routerMapsHere(pending.port, pending.protocol); + if (pendingRemovals.get(key) !== pending) return; + if (held === false) { + deletePendingRemoval(key); + log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) no longer this node's on the router, nothing to remove`); + return; + } + if (await removeProtocol(pending.port, pending.protocol)) { + deletePendingRemoval(key); + log.info(`UPnP - ${pending.protocol} ${pending.port} (${pending.description}) unmapped after ${pending.refusals} refusal(s)`); + return; + } + // mapped again while the router was being asked: in use, not pending + if (pendingRemovals.get(key) === pending) schedulePendingRemoval(pending); + }); +} - await client.removeMapping({ - public: port, - protocol: 'UDP', - }); +/** + * Asks the router again to remove a pending removal's mapping after a wait + * that doubles with each refusal, up to config.upnp.removalRetryMaxMs. + * @param {object} pending + */ +function schedulePendingRemoval(pending) { + const { removalRetryFirstMs, removalRetryMaxMs } = config.upnp; + const waitMs = Math.min(removalRetryFirstMs * (2 ** pending.refusals), removalRetryMaxMs); + // eslint-disable-next-line no-param-reassign + pending.refusals += 1; + // eslint-disable-next-line no-param-reassign + pending.timer = setTimeout(() => askPendingRemoval(pending), waitMs); +} - await serviceHelper.delay(500); +/** + * Whether the router has yet to remove a mapping of this node it was asked to. + * @returns {boolean} + */ +function removalPending() { + return pendingRemovals.size > 0; +} - return true; - } catch (error) { - log.error(error); - return false; +/** + * Removes the protocols of a port that this node mapped, each on its own: a + * refusal of one leaves the other to go. A router removes a mapping by its + * external port whoever it points at, so a protocol this node's latest map did + * not take - one another machine holds - is never asked for. A removal the + * router refuses is pending, and asked again until it goes. + * @param {number} port Port number. + * @param {string} [description] the mapping's description, for the log + * @returns {Promise} whether the router removed every one of them now + */ +async function removeMapUpnpPort(port, description = '') { + const protocols = mappedProtocols.get(+port) || new Set(); + mappedHere.delete(+port); + mappedProtocols.delete(+port); + let removed = true; + // eslint-disable-next-line no-restricted-syntax + for (const protocol of ['TCP', 'UDP']) { + // eslint-disable-next-line no-continue + if (!protocols.has(protocol)) continue; + // eslint-disable-next-line no-await-in-loop + if (!await removeProtocol(+port, protocol)) { + removed = false; + const pending = { + port: +port, protocol, description, refusals: 0, timer: null, + }; + setPendingRemoval(`${+port}/${protocol}`, pending); + schedulePendingRemoval(pending); + } + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(500); } + return removed; } /** @@ -375,9 +601,40 @@ async function removeMapUpnpPort(port) { */ function isOwnedMappingDescription(description) { if (typeof description !== 'string') return false; - if (description === FLUXADM_MAPPING_DESCRIPTION && fluxadmPort.isArcane) return false; - return OWNED_MAPPING_DESCRIPTIONS.has(description) - || OWNED_MAPPING_PREFIXES.some((prefix) => description.startsWith(prefix)); + if (sameDescription(description, FLUXADM_MAPPING_DESCRIPTION) && fluxadmPort.isArcane) return false; + return [...OWNED_MAPPING_DESCRIPTIONS].some((owned) => sameDescription(description, owned)) + || OWNED_MAPPING_PREFIXES.some((prefix) => descriptionStartsWith(description, prefix)); +} + +/** + * Whether two mapping descriptions are the same as a router stores them. + * @param {string} a + * @param {string} b + * @returns {boolean} + */ +function sameDescription(a, b) { + return natUpnp.canonicalDescription(a) === natUpnp.canonicalDescription(b); +} + +/** + * Whether a mapping description begins with a prefix, as a router stores them. + * @param {string} description + * @param {string} prefix + * @returns {boolean} + */ +function descriptionStartsWith(description, prefix) { + return natUpnp.canonicalDescription(description).startsWith(natUpnp.canonicalDescription(prefix)); +} + +/** + * Whether a mapping's description is a test's. + * @param {string} description + * @returns {boolean} + */ +function isTestMappingDescription(description) { + return typeof description === 'string' + && ([...TEST_MAPPING_DESCRIPTIONS].some((test) => sameDescription(description, test)) + || descriptionStartsWith(description, TEST_MAPPING_PREFIX)); } /** @@ -390,6 +647,15 @@ function corePorts(apiport) { return [+apiport - 1, +apiport, +apiport + 1, +apiport + 2, fluxadmPort.sshPortFor(apiport)]; } +/** + * The number of this node's cleanup slot, from its api port's tens digit. + * @param {number|string} apiport the api port this FluxOS listens on + * @returns {number} 0 to CLEANUP_SLOT_COUNT - 1 + */ +function cleanupSlotOf(apiport) { + return Math.floor((+apiport % 100) / 10) % CLEANUP_SLOT_COUNT; +} + /** * Milliseconds until the next start of this node's cleanup slot. * @param {number|string} apiport the api port this FluxOS listens on @@ -397,96 +663,41 @@ function corePorts(apiport) { * @returns {number} */ function msUntilCleanupSlot(apiport, now = Date.now()) { - const slotStart = (Math.floor((+apiport % 100) / 10) % CLEANUP_SLOT_COUNT) * CLEANUP_SLOT_MS; + const slotStart = cleanupSlotOf(apiport) * CLEANUP_SLOT_MS; return (slotStart - (now % CLEANUP_CYCLE_MS) + CLEANUP_CYCLE_MS) % CLEANUP_CYCLE_MS; } /** - * Lists the router's mappings to this node, one GetGenericPortMappingEntry per - * index, as the library's getMappings does. The walk is this code's rather than - * the library's so that it can stop: getMappings walks to the end of the table - * whatever its caller has given up on, into the next node's slot. No request - * goes out once `until` has passed (one already in flight cannot be called - * back under nat-upnp 1.x), and what was listed by then is returned: the pass - * only removes what it has seen. + * Removes the router's mappings to this node that `select` picks from them. * - * The router's error reply to an index past the last entry ends the table. No - * reply at all - a request that times out, a connection refused or reset - is a - * router in trouble, and rejects: nothing more is sent to it. - * @param {number} until monotonic ms after which no request is sent - * @param {number} deadline monotonic ms the whole pass ends at - * @returns {Promise} the mappings, in the library's getMappings shape - */ -async function listOwnMappings(until, deadline) { - const timeout = () => Math.min(CLEANUP_REQUEST_TIMEOUT_MS, deadline - monotonicMs()); - const { gateway, address } = await withinTime(client.getGateway(), timeout(), 'finding the gateway'); - const mappings = []; - for (let index = 0; ; index += 1) { - if (monotonicMs() >= until) { - log.warn(`UPnP - stale mapping cleanup out of time after ${index} router entries, the rest wait for the next start`); - return mappings; - } - let data; - try { - // eslint-disable-next-line no-await-in-loop - data = await withinTime(gateway.run('GetGenericPortMappingEntry', [['NewPortMappingIndex', index]]), timeout(), 'listing the mappings'); - } catch (error) { - if (error.response) return mappings; - throw error; - } - const key = Object.keys(data || {}).find((k) => /^GetGenericPortMappingEntryResponse/.test(k)); - if (!key) throw new Error('listing the mappings: incorrect response'); - const entry = data[key]; - if (entry.NewInternalClient === address) { - mappings.push({ - public: { host: (typeof entry.NewRemoteHost === 'string' && entry.NewRemoteHost) || '', port: parseInt(entry.NewExternalPort, 10) }, - private: { host: entry.NewInternalClient, port: parseInt(entry.NewInternalPort, 10) }, - protocol: String(entry.NewProtocol).toLowerCase(), - description: entry.NewPortMappingDescription, - }); - } - } -} - -/** - * Removes the router's mappings to this node that this code made and nothing - * here holds any more: an app removed while the router was unreachable, a - * FluxOS that died between mapping and unmapping, a node reinstalled on the - * same address without the apps it ran, an api port that moved. They are made - * with an indefinite lease (or the router's longest, a week under IGDv2), so - * nothing else ever takes them, and home routers hold only so many. - * - * One listing. Only mappings to this node's own address are looked at - - * several nodes behind one router is an ordinary setup - and only those with a - * description this code gives, on a port nothing here holds and mapUpnpPort has - * not mapped since FluxOS started. - * - * What is held is asked for only after the router has been listed: an app's - * record is written before its ports are mapped, so every mapping the listing - * holds already has its app in the answer. + * One listing, of the mappings to this node's own address only - several + * nodes behind one router is an ordinary setup. A port mapUpnpPort has mapped + * since FluxOS started, or maps while this runs, is never removed. * * Everything - the listing and each removal - happens within `budgetMs`. The - * listing stops asking the router early enough to leave time for removals, and - * what has not been listed or removed by the end is left for the next start. A - * removal the router refuses is stepped past; one it does not answer ends the - * pass, so as not to send more to a router that is not keeping up. - * @param {function(): Promise>} heldPorts every port something on this node holds + * listing has to finish early enough to leave time for removals, or nothing is + * removed; what has not been removed by the end is left for the next start. A + * removal the router refuses with a UPnP error is stepped past; any other + * failure - no answer in time, a connection refused or reset - ends the pass, + * so as not to send more to a router that is not keeping up. + * @param {function(object[]): (object[]|Promise)} select the mappings to remove, from the listing * @param {number} budgetMs + * @param {string} what the mappings' name in the log * @returns {Promise} how many mappings were removed */ -async function removeStaleMappings(heldPorts, budgetMs) { - const deadline = monotonicMs() + budgetMs; +async function removeOwnMappings(select, budgetMs, what) { + // AbortSignal.timeout takes whole milliseconds only + const wholeBudgetMs = Math.floor(budgetMs); + const deadline = monotonicMs() + wholeBudgetMs; const left = () => deadline - monotonicMs(); - const mappings = await listOwnMappings(deadline - Math.min(CLEANUP_REMOVAL_RESERVE_MS, budgetMs / 2), deadline); - const keep = new Set([...await heldPorts()].map(Number)); - const stale = mappings.filter((mapping) => Number.isInteger(mapping.public.port) - && !keep.has(mapping.public.port) - && isOwnedMappingDescription(mapping.description)); + const listingMs = wholeBudgetMs - Math.min(CLEANUP_REMOVAL_RESERVE_MS, Math.floor(wholeBudgetMs / 2)); + const mappings = await client.getMappings({ local: true, signal: AbortSignal.timeout(listingMs) }); + const chosen = (await select(mappings)).filter((mapping) => Number.isInteger(mapping.public.port)); let removed = 0; // eslint-disable-next-line no-restricted-syntax - for (const [index, mapping] of stale.entries()) { + for (const [index, mapping] of chosen.entries()) { if (left() <= 0) { - log.warn(`UPnP - stale mapping cleanup out of time, ${stale.length - index} left for the next start`); + log.warn(`UPnP - ${what} cleanup out of time, ${chosen.length - index} left for the next start`); break; } const { host, port } = mapping.public; @@ -497,13 +708,13 @@ async function removeStaleMappings(heldPorts, budgetMs) { const protocol = mapping.protocol.toUpperCase(); try { // eslint-disable-next-line no-await-in-loop - await withinTime(client.removeMapping({ public: { host, port }, protocol }), Math.min(CLEANUP_REQUEST_TIMEOUT_MS, left()), 'removing the mapping'); + await client.removeMapping({ public: { host, port }, protocol, signal: AbortSignal.timeout(left()) }); removed += 1; - log.info(`UPnP - stale mapping removed: ${protocol} ${port} (${mapping.description})`); + log.info(`UPnP - ${what} removed: ${protocol} ${port} (${mapping.description})`); } catch (error) { - log.warn(`UPnP - stale mapping ${protocol} ${port} (${mapping.description}) not removed: ${error.message}`); - if (error.timedOut) { - log.warn(`UPnP - stale mapping cleanup stopped, ${stale.length - index - 1} left for the next start`); + log.warn(`UPnP - ${what} ${protocol} ${port} (${mapping.description}) not removed: ${error.message}`); + if (!(error instanceof natUpnp.UpnpError)) { + log.warn(`UPnP - ${what} cleanup stopped, ${chosen.length - index - 1} left for the next start`); break; } } @@ -513,6 +724,45 @@ async function removeStaleMappings(heldPorts, budgetMs) { return removed; } +/** + * Removes the router's mappings to this node that this code made and nothing + * here holds any more: an app removed while the router was unreachable, a + * FluxOS that died between mapping and unmapping, a node reinstalled on the + * same address without the apps it ran, an api port that moved. They are made + * with an indefinite lease (or the router's longest, a week under IGDv2), so + * nothing else ever takes them, and home routers hold only so many. + * + * Only those with a description this code gives, on a port nothing here + * holds. What is held is asked for only after the router has been listed: an + * app's record is written before its ports are mapped, so every mapping the + * listing holds already has its app in the answer. + * @param {function(): Promise>} heldPorts every port something on this node holds + * @param {number} budgetMs + * @returns {Promise} how many mappings were removed + */ +async function removeStaleMappings(heldPorts, budgetMs) { + return removeOwnMappings(async (mappings) => { + const keep = new Set([...await heldPorts()].map(Number)); + return mappings.filter((mapping) => !keep.has(mapping.public.port) + && isOwnedMappingDescription(mapping.description)); + }, budgetMs, 'stale mapping'); +} + +/** + * Removes the router's test mappings to this node: a port test's, an app's + * prelaunch test's and verifyUPNPsupport's. A test removes its mapping before + * it ends, so while UPnP is being verified at the start of FluxOS every one the + * router holds was left by an earlier run. + * @returns {Promise} how many mappings were removed + */ +async function removeLeftoverTestMappings() { + return removeOwnMappings( + (mappings) => mappings.filter((mapping) => isTestMappingDescription(mapping.description)), + CLEANUP_SLOT_MS, + 'left-over test mapping', + ); +} + /** * To map a specified port and show a message if successfully mapped. Only accessible by admins and Flux team members. * @param {object} req Request. @@ -663,8 +913,12 @@ async function getGatewayApi(req, res) { try { const authorized = await verificationHelper.verifyPrivilege(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)); if (authorized) { - const gateway = await client.getGateway(); - const message = messageHelper.createDataMessage(gateway); + const info = await client.getGateway(); + const { description, services } = info.gateway; + const message = messageHelper.createDataMessage({ + gateway: { description, services }, + address: await info.getLocalAddress(), + }); res.json(message); } else { const errMessage = messageHelper.errUnauthorizedMessage(); @@ -687,11 +941,17 @@ module.exports = { setupUPNP, mapUpnpPort, removeMapUpnpPort, + routerRefusals, corePorts, msUntilCleanupSlot, removeStaleMappings, mappedHere, + mappedProtocols, + pendingRemovals, + removalPending, + routerEvents, CLEANUP_SLOT_MS, + BOOT_LISTING_STAGGER_MS, mapPortApi, removeMapPortApi, getMapApi, diff --git a/ZelBack/src/services/utils/appConstants.js b/ZelBack/src/services/utils/appConstants.js index e19b21934e..6facb02a2f 100644 --- a/ZelBack/src/services/utils/appConstants.js +++ b/ZelBack/src/services/utils/appConstants.js @@ -139,14 +139,6 @@ const ANNOUNCE_INTERVAL_MS = Math.floor( (RUNNING_EXPIRY_MS * (1 - ANNOUNCE_SLACK)) / ANNOUNCES_PER_EXPIRY / 1000, ) * 1000; -// How long a removal waits for an announcement cycle to finish sending before it -// tells the network the app is gone. A cycle is one database read per installed -// app, one signature and one broadcast, so this is orders of magnitude above a -// healthy one at any plausible app count - and it is bounded at all because a -// wedged cycle must not hold up the node's removals, which take the removal lock -// and block every install and redeploy behind them. -const ANNOUNCE_CYCLE_WAIT_MS = 30 * 1000; - // Hash sync constants (blocks, at 30s per block) const HASH_EXPIRY_BLOCKS = 1051200; // ~1 year — permanently flag unresolvable hashes const HASH_RETRY_BACKOFF = [0, 100, 500, 2500, 12500, 50000, 100000]; // ~0, 50min, 4h, 21h, 4d, 17d, 35d @@ -195,7 +187,6 @@ module.exports = { GOSSIP_VALIDITY_MS, RUNNING_EXPIRY_MS, ANNOUNCE_INTERVAL_MS, - ANNOUNCE_CYCLE_WAIT_MS, INSTALLING_EXPIRY_MS, INSTALLING_ERRORS_EXPIRY_MS, SIGTERM_EXPIRY_MS, diff --git a/ZelBack/src/services/utils/appIdentity.js b/ZelBack/src/services/utils/appIdentity.js new file mode 100644 index 0000000000..a761059185 --- /dev/null +++ b/ZelBack/src/services/utils/appIdentity.js @@ -0,0 +1,32 @@ +const signatureVerifier = require('../signatureVerifier'); +const { APP_NAME_REGEX, MAX_APP_NAME_LENGTH } = require('./appConstants'); + +/** + * Whether an app name and owner are well formed: a name as a spec allows it, + * and an owner that can sign (a Flux ID or an Ethereum address). + * @param {*} name App name. + * @param {*} owner App owner. + * @returns {boolean} + */ +function isValidAppNameAndOwner(name, owner) { + return typeof name === 'string' + && name.length <= MAX_APP_NAME_LENGTH + && APP_NAME_REGEX.test(name) + && signatureVerifier.isValidSigningIdentity(owner); +} + +/** + * Throws unless the app name and owner are well formed. + * @param {*} name App name. + * @param {*} owner App owner. + */ +function verifyAppNameAndOwner(name, owner) { + if (!isValidAppNameAndOwner(name, owner)) { + throw new Error('Invalid app name or owner'); + } +} + +module.exports = { + isValidAppNameAndOwner, + verifyAppNameAndOwner, +}; diff --git a/ZelBack/src/services/utils/appRunningEntries.js b/ZelBack/src/services/utils/appRunningEntries.js new file mode 100644 index 0000000000..6b677f332c --- /dev/null +++ b/ZelBack/src/services/utils/appRunningEntries.js @@ -0,0 +1,25 @@ +/** + * The app entries of a version 2 apprunning message that can be stored, and the + * ones that cannot. An entry is a location row only with a string name and a + * string hash. Entries are judged one at a time, so an entry that fails costs + * its sender that entry and not the rest of the message. + * + * The announcer applies the same rule to what it sends, so a node never + * announces an entry its peers refuse. + * @param {Array<*>} apps The message's apps. + * @returns {{stored: Array, refused: Array<*>}} Storable entries and the rest. + */ +function appRunningEntries(apps) { + const stored = []; + const refused = []; + for (const app of apps) { + if (app && typeof app === 'object' && typeof app.name === 'string' && typeof app.hash === 'string') { + stored.push(app); + } else { + refused.push(app); + } + } + return { stored, refused }; +} + +module.exports = { appRunningEntries }; diff --git a/ZelBack/src/services/utils/appSpecHelpers.js b/ZelBack/src/services/utils/appSpecHelpers.js index cbc5b605d1..3177eee7a4 100644 --- a/ZelBack/src/services/utils/appSpecHelpers.js +++ b/ZelBack/src/services/utils/appSpecHelpers.js @@ -5,6 +5,7 @@ const messageHelper = require('../messageHelper'); const serviceHelper = require('../serviceHelper'); const daemonServiceMiscRpcs = require('../daemonService/daemonServiceMiscRpcs'); const { checkAndDecryptAppSpecs } = require('./enterpriseHelper'); +const { verifyAppNameAndOwner } = require('./appIdentity'); const { appPricePerMonth } = require('./appUtilities'); const { specificationFormatter } = require('./appUtilities'); const { getChainParamsPriceUpdates } = require('./chainUtilities'); @@ -325,6 +326,9 @@ async function getAppFiatAndFluxPrice(req, res) { throw new Error('Daemon not yet synced.'); } const daemonHeight = syncStatus.data.height; + if (appSpecification.version >= 8 && appSpecification.enterprise) { + verifyAppNameAndOwner(appSpecification.name, appSpecification.owner); + } appSpecification = await checkAndDecryptAppSpecs(appSpecification, { daemonHeight }); const appSpecFormatted = specificationFormatter(appSpecification); diff --git a/ZelBack/src/services/utils/appSyncEvents.js b/ZelBack/src/services/utils/appSyncEvents.js index aec590dda0..7df3ce9c8f 100644 --- a/ZelBack/src/services/utils/appSyncEvents.js +++ b/ZelBack/src/services/utils/appSyncEvents.js @@ -6,11 +6,15 @@ const EVENTS = Object.freeze({ EPHEMERAL_SYNC_COMPLETE: 'ephemeralSyncComplete', EPHEMERAL_SYNC_REFUSED: 'ephemeralSyncRefused', EPHEMERAL_SYNC_UNVERIFIED: 'ephemeralSyncUnverified', + EPHEMERAL_SYNC_WRITE_FAILED: 'ephemeralSyncWriteFailed', EPHEMERAL_SYNC_PROGRESS: 'ephemeralSyncProgress', SPAWNER_READY: 'spawnerReady', READINESS_LOST: 'readinessLost', HASH_RESPONSE_RECEIVED: 'hashResponseReceived', HASH_UNRESOLVED: 'hashUnresolved', + // An operator asked for a check of every missing app message; the orchestrator + // runs it once no hash sync or registry build holds the node. + HASH_CHECK_REQUESTED: 'hashCheckRequested', }); module.exports = { appSyncEvents, EVENTS }; diff --git a/ZelBack/src/services/utils/appUtilities.js b/ZelBack/src/services/utils/appUtilities.js index 318ad2a933..edd75bf6f0 100644 --- a/ZelBack/src/services/utils/appUtilities.js +++ b/ZelBack/src/services/utils/appUtilities.js @@ -1198,6 +1198,13 @@ function parseContainerName(containerName) { }; } +/** + * Whether this node holds an unexpired location record for an app. + * @param {string} appName + * @param {string} localSocketAddr + * @returns {Promise} null when the records cannot be read: an unread record is + * neither a valid nor an expired one. + */ async function appHasValidLocationOnNode(appName, localSocketAddr) { try { const db = dbHelper.databaseConnection(); @@ -1215,7 +1222,7 @@ async function appHasValidLocationOnNode(appName, localSocketAddr) { }); } catch (error) { log.error(`Error checking app location for ${appName}: ${error.message}`); - return true; + return null; } } diff --git a/ZelBack/src/services/utils/chainUtilities.js b/ZelBack/src/services/utils/chainUtilities.js index 903c346091..6ce60d1358 100644 --- a/ZelBack/src/services/utils/chainUtilities.js +++ b/ZelBack/src/services/utils/chainUtilities.js @@ -4,51 +4,47 @@ const log = require('../../lib/log'); /** * To get array of price specifications updates - * @returns {(object|object[])} Returns an array of app objects. + * @returns {Promise} The price forks, oldest first: the configured ones and the chain's. + * Rejects when the chain's cannot be read - a list without them prices against stale forks. */ async function getChainParamsPriceUpdates() { - try { - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.chainparams.database); - const chainParamsMessagesCollection = config.database.chainparams.collections.chainMessages; - const query = { version: 'p' }; - const projection = { - projection: { - _id: 0, - }, - }; - const priceMessages = await dbHelper.findInDatabase(database, chainParamsMessagesCollection, query, projection); - const priceForks = []; - config.fluxapps.price.forEach((price) => { - priceForks.push(price); - }); - priceMessages.forEach((data) => { - const splittedMess = data.message.split('_'); - if (splittedMess[4]) { - const dataPoint = { - height: +data.height, - cpu: +splittedMess[1], - ram: +splittedMess[2], - hdd: +splittedMess[3], - minPrice: +splittedMess[4], - port: +splittedMess[5] || 2, - scope: +splittedMess[6] || 6, - staticip: +splittedMess[7] || 3, - }; - priceForks.push(dataPoint); - } - }); - // sort priceForks depending on height - priceForks.sort((a, b) => { - if (a.height > b.height) return 1; - if (a.height < b.height) return -1; - return 0; - }); - return priceForks; - } catch (error) { - log.error(error); - return []; - } + const db = dbHelper.databaseConnection(); + const database = db.db(config.database.chainparams.database); + const chainParamsMessagesCollection = config.database.chainparams.collections.chainMessages; + const query = { version: 'p' }; + const projection = { + projection: { + _id: 0, + }, + }; + const priceMessages = await dbHelper.findInDatabase(database, chainParamsMessagesCollection, query, projection); + const priceForks = []; + config.fluxapps.price.forEach((price) => { + priceForks.push(price); + }); + priceMessages.forEach((data) => { + const splittedMess = data.message.split('_'); + if (splittedMess[4]) { + const dataPoint = { + height: +data.height, + cpu: +splittedMess[1], + ram: +splittedMess[2], + hdd: +splittedMess[3], + minPrice: +splittedMess[4], + port: +splittedMess[5] || 2, + scope: +splittedMess[6] || 6, + staticip: +splittedMess[7] || 3, + }; + priceForks.push(dataPoint); + } + }); + // sort priceForks depending on height + priceForks.sort((a, b) => { + if (a.height > b.height) return 1; + if (a.height < b.height) return -1; + return 0; + }); + return priceForks; } /** diff --git a/ZelBack/src/services/utils/cpuBurstHelper.js b/ZelBack/src/services/utils/cpuBurstHelper.js index c704fbe79d..0686a37079 100644 --- a/ZelBack/src/services/utils/cpuBurstHelper.js +++ b/ZelBack/src/services/utils/cpuBurstHelper.js @@ -6,8 +6,6 @@ const log = require('../../lib/log'); const enterpriseConfig = require('./enterpriseConfig'); const signatureVerifier = require('../signatureVerifier'); -let burstSupportCache = null; - /** * Checks if an app owner is in the enterprise app owners whitelist. An unknown policy * answers false: burst is a privilege, and one that cannot be verified is not granted. @@ -21,52 +19,12 @@ function isEnterpriseOwner(owner) { } /** - * Checks if the system supports CFS CPU burst (cgroups v2 + kernel >= 5.14). - * Caches the result after first check. - * @returns {Promise} + * Whether CPU burst is switched on. Every node FluxOS runs apps on has what it + * needs: cgroup v2 and Linux 5.14 (fluxNetworkHelper.checkCgroupsAllowed). + * @returns {boolean} */ -async function isCpuBurstSupported() { - if (burstSupportCache !== null) return burstSupportCache; - - try { - if (!config.cpuBurst?.enabled) { - burstSupportCache = false; - return false; - } - - // Check cgroups v2: /sys/fs/cgroup/cgroup.controllers must exist - try { - await fsp.access('/sys/fs/cgroup/cgroup.controllers'); - } catch { - log.warn('CPU burst: cgroups v2 not detected, burst disabled'); - burstSupportCache = false; - return false; - } - - // Check kernel version >= 5.14 for cpu.cfs_burst_us support - const release = await fsp.readFile('/proc/version', 'utf8'); - const match = release.match(/(\d+)\.(\d+)/); - if (!match) { - log.warn('CPU burst: unable to parse kernel version, burst disabled'); - burstSupportCache = false; - return false; - } - const major = parseInt(match[1], 10); - const minor = parseInt(match[2], 10); - if (major < 5 || (major === 5 && minor < 14)) { - log.warn(`CPU burst: kernel ${major}.${minor} < 5.14, burst disabled`); - burstSupportCache = false; - return false; - } - - burstSupportCache = true; - log.info('CPU burst: system supports CFS burst (cgroups v2, kernel >= 5.14)'); - return true; - } catch (error) { - log.warn(`CPU burst: capability check failed: ${error.message}`); - burstSupportCache = false; - return false; - } +function isCpuBurstEnabled() { + return config.cpuBurst?.enabled === true; } /** @@ -185,7 +143,7 @@ async function isBurstActive(pid) { */ async function applyBurst(pid, cpuCores, identifier) { if (!cpuCores || cpuCores <= 0) return false; - if (!await isCpuBurstSupported()) return false; + if (!isCpuBurstEnabled()) return false; const { burstUs } = calculateBurstParams(cpuCores); const ok = await setCpuBurst(pid, burstUs); @@ -198,20 +156,12 @@ async function applyBurst(pid, cpuCores, identifier) { return ok; } -/** - * Resets the burst support cache. Used in testing. - */ -function resetBurstSupportCache() { - burstSupportCache = null; -} - module.exports = { isEnterpriseOwner, - isCpuBurstSupported, + isCpuBurstEnabled, getCgroupBurstPath, calculateBurstParams, setCpuBurst, isBurstActive, applyBurst, - resetBurstSupportCache, }; diff --git a/ZelBack/src/services/utils/cpuThrottleExemption.js b/ZelBack/src/services/utils/cpuThrottleExemption.js new file mode 100644 index 0000000000..5e62bc9513 --- /dev/null +++ b/ZelBack/src/services/utils/cpuThrottleExemption.js @@ -0,0 +1,99 @@ +const log = require('../../lib/log'); +const policyStore = require('../policyStore'); + +// Which apps the CPU throttle leaves at their full specified allocation, read from the signed +// policy bundle. The document lists app name prefixes: the ones the games hub, and the +// dedicated sites before it, write for the game servers they deploy (mostly the lowercased +// marketplace name). Every one of them names a server ``, so a prefix alone +// is not enough to qualify -- the rest of the name must be exactly that millisecond timestamp, +// and a plausible one. +// +// Game servers are exempt because a game's tick is a latency budget, not a throughput job. +// Taking 20% of a saturated server's cpu does not slow it down evenly, it makes every tick that +// overruns the budget arrive late, which players feel as lag and rubber-banding. The throttle +// then never lets go: it measures load against the reduced allocation, so a server that was +// saturated stays saturated, and stays at 80%. +// +// Exemption only ever holds an app at what its owner specified and paid for. It never grants +// more, so a name that qualifies without being a game gains nothing it was not already owed. +// +// Unknown policy answers "not exempt": the throttle is the default, and a node that has not read +// policy keeps doing what it did before this document existed. + +const DOCUMENT = 'cputhrottleexempt'; + +// Date.now() in milliseconds is 13 digits from 2001 until the year 2286. +const TIMESTAMP_DIGITS = 13; +// The earliest game server the hub, or the dedicated sites before it, could have named. +const EARLIEST_TIMESTAMP_MS = Date.UTC(2023, 0, 1); +// A name is minted in the customer's browser before registration, so its timestamp can sit a +// little ahead of this node's clock. A day absorbs any skew; beyond that it is not a timestamp. +const FUTURE_TOLERANCE_MS = 24 * 60 * 60 * 1000; + +const PREFIX_PATTERN = /^[a-z0-9]+$/; + +// Memoized prefix set, keyed by the document it was built from. The store hands back the same +// object until it adopts a new bundle, so reference identity is a sound signal. +let prefixCache = null; +let prefixCacheKey = null; + +/** + * A valid document is an array of lowercase alphanumeric prefixes. + * + * Still checked even though the bundle is signed: a signature says who published a document, + * not that it is the shape this code expects. + */ +function isValidPrefixList(value) { + return Array.isArray(value) + && value.every((prefix) => typeof prefix === 'string' && PREFIX_PATTERN.test(prefix)); +} + +/** + * The exempt prefixes as a Set, or null when the policy is unknown or malformed. + * @returns {Set|null} + */ +function getExemptPrefixes() { + const document = policyStore.getDocument(DOCUMENT); + if (document === null) return null; + if (prefixCacheKey === document) return prefixCache; + // The verdict is cached either way. A malformed document is asked about once per app per + // throttle pass, and logging each time would bury the one line that matters. + prefixCacheKey = document; + if (!isValidPrefixList(document)) { + log.error(`cpuThrottleExemption - ${DOCUMENT} in the signed bundle is not an array of lowercase alphanumeric prefixes`); + prefixCache = null; + return null; + } + prefixCache = new Set(document); + return prefixCache; +} + +/** + * Whether an app is a hub game server the CPU throttle must leave alone. + * + * The timestamp is the last 13 characters, and the prefix is everything before them. Splitting + * from the end rather than matching the prefix first is what keeps a prefix that itself ends in + * a digit (eurotrucksimulator2) unambiguous, and keeps one prefix from claiming another's apps + * (rustserver does not cover rustserveroxide; each is listed on its own). + * @param {string} appName - The application name + * @param {number} [now] - The current time in ms, for tests + * @returns {boolean} + */ +function isCpuThrottleExempt(appName, now = Date.now()) { + if (typeof appName !== 'string' || appName.length <= TIMESTAMP_DIGITS) return false; + const prefixes = getExemptPrefixes(); + if (prefixes === null || prefixes.size === 0) return false; + + const name = appName.toLowerCase(); + const timestamp = name.slice(-TIMESTAMP_DIGITS); + if (!/^\d+$/.test(timestamp)) return false; + const ms = Number(timestamp); + if (ms < EARLIEST_TIMESTAMP_MS || ms > now + FUTURE_TOLERANCE_MS) return false; + + return prefixes.has(name.slice(0, -TIMESTAMP_DIGITS)); +} + +module.exports = { + getExemptPrefixes, + isCpuThrottleExempt, +}; diff --git a/ZelBack/src/services/utils/enterpriseHelper.js b/ZelBack/src/services/utils/enterpriseHelper.js index 2b556ff760..9831d76d2d 100644 --- a/ZelBack/src/services/utils/enterpriseHelper.js +++ b/ZelBack/src/services/utils/enterpriseHelper.js @@ -146,7 +146,7 @@ async function decryptEnterpriseFromSession(base64Encrypted, appName, daemonHeig * @returns {Promise} Decrypted specifications */ async function checkAndDecryptAppSpecs(appSpec, options = {}) { - if (!appSpec || appSpec.version < 8 || !appSpec.enterprise) { + if (!appSpec || !(appSpec.version >= 8 && appSpec.enterprise)) { return appSpec; } @@ -158,7 +158,6 @@ async function checkAndDecryptAppSpecs(appSpec, options = {}) { const appSpecs = JSON.parse(JSON.stringify(appSpec)); let daemonHeight = options.daemonHeight || null; - let appOwner = options.owner || null; const db = dbHelper.databaseConnection(); const database = db.db(config.database.appsglobal.database); @@ -170,21 +169,6 @@ async function checkAndDecryptAppSpecs(appSpec, options = {}) { }; let appsQuery = null; - if (!appOwner) { - log.info(`Searching register permanent messages for ${appSpecs.name} to get registration message`); - appsQuery = { - 'appSpecifications.name': appSpecs.name, - type: 'fluxappregister', - }; - const permanentAppMessage = await dbHelper.findInDatabase(database, globalAppsMessages, appsQuery, projection); - if (permanentAppMessage.length > 0) { - const lastAppRegistration = permanentAppMessage[permanentAppMessage.length - 1]; - appOwner = lastAppRegistration.owner; - } else { - appOwner = appSpec.owner; - } - } - if (!daemonHeight) { log.info(`Searching register permanent messages for ${appSpecs.name} to get latest update`); appsQuery = { @@ -281,80 +265,8 @@ async function encryptEnterpriseFromSession(appSpec, daemonHeight, enterpriseKey throw new Error('Error encrypting enterprise object.'); } -/** - * Encrypts enterprise content with AES - * @param {object} enterprise - Content to be encrypted - * @param {string} appName - Application name - * @param {number} daemonHeight - Daemon block height (optional) - * @param {string} owner - Application owner (optional) - * @returns {Promise} Encrypted enterprise content - */ -async function encryptEnterpriseWithAes(enterprise, appName, daemonHeight = null, owner = null) { - let block = daemonHeight; - let appOwner = owner; - - if (!isArcane) { - throw new Error('Application Specifications can only be validated on a node running Arcane OS.'); - } - - const db = dbHelper.databaseConnection(); - const database = db.db(config.database.appsglobal.database); - const globalAppsMessages = config.database.appsglobal.collections.appsMessages; - const projection = { - projection: { - _id: 0, - }, - }; - let appsQuery = null; - - if (!appOwner) { - log.info(`Searching register permanent messages for ${appName} to get registration message`); - appsQuery = { - 'appSpecifications.name': appName, - type: 'fluxappregister', - }; - const permanentAppMessage = await dbHelper.findInDatabase(database, globalAppsMessages, appsQuery, projection); - const lastAppRegistration = permanentAppMessage[permanentAppMessage.length - 1]; - appOwner = lastAppRegistration.owner; - } - - if (!block) { - log.info(`Searching register permanent messages for ${appName} to get latest update`); - appsQuery = { - 'appSpecifications.name': appName, - }; - const allPermanentAppMessage = await dbHelper.findInDatabase(database, globalAppsMessages, appsQuery, projection); - const lastUpdate = allPermanentAppMessage[allPermanentAppMessage.length - 1]; - block = lastUpdate.height; - } - - const jsonEnterprise = JSON.stringify(enterprise); - const base64JsonEnterprise = Buffer.from(jsonEnterprise).toString('base64'); - - const inputData = JSON.stringify({ - fluxID: appOwner, - appName, - message: base64JsonEnterprise, - blockHeight: block, - }); - - const dataReturned = await benchmarkService.encryptMessage(inputData); - const { status, data } = dataReturned; - if (status === 'success') { - const dataParsed = JSON.parse(data); - const newEnterprise = status === 'success' && dataParsed.status === 'ok' ? dataParsed.message : null; - if (newEnterprise) { - return newEnterprise; - } - throw new Error('Error encrypting application specifications.'); - } else { - throw new Error('Error getting encrypted specifications.'); - } -} - module.exports = { checkAndDecryptAppSpecs, - encryptEnterpriseWithAes, encryptEnterpriseFromSession, decryptEnterpriseFromSession, decryptAesKeyWithRsaKey, diff --git a/ZelBack/src/services/utils/fluxEventBus.js b/ZelBack/src/services/utils/fluxEventBus.js index d82b3d9f7b..b3d3451ac3 100644 --- a/ZelBack/src/services/utils/fluxEventBus.js +++ b/ZelBack/src/services/utils/fluxEventBus.js @@ -68,6 +68,22 @@ const Checkpoint = Object.freeze({ // The elected leader has read its sync status and has not yet checked the disk // it would seed from holds what that status claims. Keyed by folder id. SYNCTHING_BEFORE_SEED_CHECK: 'syncthing:beforeSeedCheck', + // A check of a receive-only folder's local changes has been started and has not + // read anything. Keyed by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_CHECK: 'syncthing:beforeLocalChangeCheck', + // A check of a receive-only folder's local changes has its answer and has not + // left it for the monitor pass. Keyed by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_ANSWER: 'syncthing:beforeLocalChangeAnswer', + // A revert of a receive-only folder's local changes has not yet read whether they + // are still the ones checked. Keyed by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT: 'syncthing:beforeLocalChangeRevert', + // A revert of a receive-only folder's local changes holds the folder's config, + // has found the folder the one checked, and has not yet sent the revert. Keyed + // by folder id. + SYNCTHING_BEFORE_LOCAL_CHANGE_REVERT_SENT: 'syncthing:beforeLocalChangeRevertSent', + // The syncthing monitor has reached a receive-only folder and not yet measured + // what this node holds in it. Keyed by folder id. + SYNCTHING_BEFORE_HOLDINGS: 'syncthing:beforeHoldings', // The syncthing monitor has found a single-writer folder sending on a node // that does not hold it, and has not yet asked for it to be held as a // standby. Keyed by folder id. @@ -87,6 +103,9 @@ const Checkpoint = Object.freeze({ // This node has recorded a move of its own address and has not yet acted on // it. Keyed by the address it moved to. OWN_ADDRESS_BEFORE_RECONCILE: 'ownAddress:beforeReconcile', + // Boot has read the location record of an app it held unconfirmed, and has not + // yet released the app or acted on the answer. Keyed by app name. + BOOT_BEFORE_LOCATION_DECISION: 'boot:beforeLocationDecision', }); const CHECKPOINT_NAMES = new Set(Object.values(Checkpoint)); // Holds a key for every key of its checkpoint. diff --git a/ZelBack/src/services/utils/fluxUdpTestServer.js b/ZelBack/src/services/utils/fluxUdpTestServer.js new file mode 100644 index 0000000000..5bf9f0c526 --- /dev/null +++ b/ZelBack/src/services/utils/fluxUdpTestServer.js @@ -0,0 +1,119 @@ +const dgram = require('node:dgram'); +const { once } = require('node:events'); + +/** + * Every probe a peer sends to a UDP port under test starts with this, and every + * reply from a test server starts with it too. + */ +const UDP_PROBE_MAGIC = 'FLUXPT'; + +/** + * The size of every probe. A reply is never larger: the test server answers a + * stranger who spoofs a peer's address, so it must not amplify what it is sent. + */ +const UDP_PROBE_BYTES = 64; + +/** + * The most replies a test server sends to one asked peer. A peer sends a + * handful of datagrams per port per attempt; this bounds what a flood of + * probes forged from the asked peer's address can make the server send, and + * a flood forged from one peer's address leaves the next peer its own replies. + */ +const MAX_REPLIES = 32; + +function noop() {} + +/** + * The reply the test server sends for a token. + * @param {string} token + * @returns {Buffer} + */ +function udpReply(token) { + return Buffer.from(`${UDP_PROBE_MAGIC} ${token}`); +} + +/** + * The UDP half of a port test: bound on the port under test while the test + * runs, and answering a peer's probe with the run's secret token. + * + * It answers only the peer this node is asking at the moment, set with + * `allow`. A UDP reply goes wherever the probe claims it came from, so + * answering anyone would make every port under test a reflector. + */ +class FluxUdpTestServer { + #socket = dgram.createSocket('udp4'); + + #reply = null; + + #allowed = null; + + #replies = 0; + + /** + * @param {string} token The secret this run's test servers answer with. + */ + constructor(token) { + this.#reply = udpReply(token); + if (this.#reply.length > UDP_PROBE_BYTES) { + throw new Error(`A UDP test token of ${token.length} characters does not fit a ${UDP_PROBE_BYTES} byte probe`); + } + this.#socket.on('message', (message, rinfo) => this.#answer(message, rinfo)); + // A bind error rejects listen(); a send that fails costs that one reply. + // Neither may reach the process as an unhandled 'error'. + this.#socket.on('error', noop); + } + + /** + * The peer whose probes this server answers from now on, with a fresh reply + * cap; null answers nobody. + * @param {string|null} ip + */ + allow(ip) { + this.#allowed = ip ? ip.replace(/^::ffff:/i, '') : null; + this.#replies = 0; + } + + /** + * Binds the port. Rejects with the bind error, EADDRINUSE among them. + * @param {number} port + * @param {string} [address] + * @returns {Promise} + */ + async listen(port, address) { + const bound = once(this.#socket, 'listening'); + this.#socket.bind(port, address); + await bound; + } + + /** @returns {{address: string, port: number}} */ + address() { + return this.#socket.address(); + } + + /** Unbinds the port. Safe to call on a server that never bound or already closed. */ + close() { + try { + this.#socket.close(); + } catch { + // ERR_SOCKET_DGRAM_NOT_RUNNING: closed already + } + } + + #answer(message, rinfo) { + if (this.#replies >= MAX_REPLIES) return; + if (!this.#allowed || rinfo.address.replace(/^::ffff:/i, '') !== this.#allowed) return; + if (message.length < UDP_PROBE_BYTES) return; + if (message.subarray(0, UDP_PROBE_MAGIC.length).toString('latin1') !== UDP_PROBE_MAGIC) return; + + this.#replies += 1; + this.#socket.send(this.#reply, rinfo.port, rinfo.address, noop); + } +} + +module.exports = { + FluxUdpTestServer, + UDP_PROBE_MAGIC, + UDP_PROBE_BYTES, + MAX_REPLIES, + udpReply, +}; diff --git a/ZelBack/src/services/utils/globalState.js b/ZelBack/src/services/utils/globalState.js index a73bc2d960..64e0c26125 100644 --- a/ZelBack/src/services/utils/globalState.js +++ b/ZelBack/src/services/utils/globalState.js @@ -124,6 +124,18 @@ const departingApps = { }, }; +// Apps boot found stopped and whose location records it could not read, so it cannot +// tell whether this node still holds them or the network placed them elsewhere while +// it was down. Until boot reads the records, an app here is not started, elected, +// announced, backed up, restored or torn down to be rebuilt. +// +// Keyed by app name; the value is the hold, so the check deciding an app can tell +// its own hold from a later one under the same name. An app's full uninstall ends +// its hold. +// +// In-memory deliberately: a restart runs the boot check again. +const unconfirmedApps = new Map(); + // Containers intentionally stopped by FluxOS — crash recovery skips die events for these const stoppingContainers = new Set(); @@ -165,12 +177,11 @@ const fluxRemovedContainers = new Set(); // once. Same null-is-no-opinion convention appReconciler's controllerDesired uses. let promotedFolderIds = null; -// What each receive-only folder on this node holds that the cluster's index does not: -// folderId -> { bytes, newestModified }. Peers ask for it before promoting one of their -// own, so that the node with the data wins rather than the node with the lowest address. -// Null until the first monitor pass, for the same reason promotedFolderIds is: "I hold -// nothing" and "I have not looked" are opposite answers to a peer about to seed. -let folderHoldings = null; +// Where each receive-only folder of an installed app stands in telling peers what this +// node holds of it: folderId -> a state from appMonitoring/folderStates. Peers read it +// before promoting one of their own, so that the node with the data wins rather than +// the node with the lowest address. +const folderStates = new Map(); // This node's place in deciding which holder seeds a folder at a cold start: // folderId -> { stage, bytes, newestModified }. `intent` while this node is elected, @@ -355,14 +366,14 @@ module.exports = { get receiveOnlySyncthingAppsCache() { return receiveOnlySyncthingAppsCache; }, get promotedFolderIds() { return promotedFolderIds; }, set promotedFolderIds(ids) { promotedFolderIds = ids; }, - get folderHoldings() { return folderHoldings; }, - set folderHoldings(map) { folderHoldings = map; }, + get folderStates() { return folderStates; }, get seedMarks() { return seedMarks; }, get syncthingDevicesIDCache() { return syncthingDevicesIDCache; }, get folderHealthCache() { return folderHealthCache; }, get runningAppsCache() { return runningAppsCache; }, get departingApps() { return departingApps; }, get testInstallingApps() { return testInstallingApps; }, + get unconfirmedApps() { return unconfirmedApps; }, get announceCycle() { return announceCycle; }, get stoppingContainers() { return stoppingContainers; }, get fluxRemovedContainers() { return fluxRemovedContainers; }, diff --git a/ZelBack/src/services/utils/installOutcome.js b/ZelBack/src/services/utils/installOutcome.js index 7248edcec1..81c18e55e8 100644 --- a/ZelBack/src/services/utils/installOutcome.js +++ b/ZelBack/src/services/utils/installOutcome.js @@ -8,13 +8,13 @@ * the response stream left to tell them apart. * * INSTALLED and ALREADY_INSTALLED both mean the node holds the app. DECLINED and - * FAILED both mean it does not. BUSY says nothing about the app at all - only about - * the node at that instant. + * FAILED both mean it does not. BUSY and DEFERRED say nothing about the app at all - + * only about the node at that instant. * * Every outcome except BUSY is returned after the install hold is acquired, so each * is a true statement about the app whoever asked for it: concurrent attempts are - * serialised, and the ones that lose get BUSY. BUSY is therefore the only outcome a - * caller must keep waiting through rather than act on. + * serialised, and the ones that lose get BUSY. BUSY and DEFERRED are therefore the + * outcomes a caller must keep waiting through rather than act on. * * Internal only - nothing answers these to a client, so unlike Privilege the values * are ours as well as the names. They are strings rather than booleans so a call @@ -39,6 +39,9 @@ const InstallOutcome = Object.freeze({ // The install got part way and cleaned up after itself, so the app is no longer on // this node. The only outcome that justifies a caller acting on the loss. FAILED: 'failed', + // The node's database did not answer, so the attempt stopped where it was and + // tore nothing down. Carries no claim about the app: a later attempt decides. + DEFERRED: 'deferred', }); module.exports = { InstallOutcome }; diff --git a/ZelBack/src/services/utils/mountParser.js b/ZelBack/src/services/utils/mountParser.js index b6ea7b4752..d9de884a29 100644 --- a/ZelBack/src/services/utils/mountParser.js +++ b/ZelBack/src/services/utils/mountParser.js @@ -530,17 +530,47 @@ function isGComponent(containerData) { } /** - * The component of an app whose start the primary election decides: its first - * g: component, as `_` (the app name for a v1-3 app), or null - * when it has none. The election decides one component per app, this one. + * The components of an app whose start the primary election decides: every g: + * component, in compose order, as `_` (the app name for a v1-3 + * app). Empty when it has none. The election decides the app, and its primary + * runs all of them. + * @param {object} appSpec - Installed app specification, decrypted. + * @returns {string[]} + */ +function gComponentIdentifiers(appSpec) { + if (!appSpec) return []; + if (appSpec.version <= 3) return isGComponent(appSpec.containerData) ? [appSpec.name] : []; + return (appSpec.compose || []) + .filter((comp) => isGComponent(comp.containerData)) + .map((comp) => `${comp.name}_${appSpec.name}`); +} + +/** + * The components of an app whose data is synced (g:/r:/s:), in compose order, as + * `_` (the app name for a v1-3 app). Empty when it has none. + * @param {object} appSpec - Installed app specification, decrypted. + * @returns {string[]} + */ +function syncedComponentIdentifiers(appSpec) { + if (!appSpec) return []; + if (appSpec.version <= 3) return isSyncedComponent(appSpec.containerData) ? [appSpec.name] : []; + return (appSpec.compose || []) + .filter((comp) => isSyncedComponent(comp.containerData)) + .map((comp) => `${comp.name}_${appSpec.name}`); +} + +/** + * An app's anchor: its first g: component, or null when it has none. + * + * A FluxOS release that predates electing every g: component knows only this + * one, and asks a peer whether it holds the app by asking about it - so a node + * holding any g: component of the app reports holding the anchor + * (appQueryService.heldComponents). * @param {object} appSpec - Installed app specification, decrypted. * @returns {string|null} */ -function electedComponentIdentifier(appSpec) { - if (!appSpec) return null; - if (appSpec.version <= 3) return isGComponent(appSpec.containerData) ? appSpec.name : null; - const component = (appSpec.compose || []).find((comp) => isGComponent(comp.containerData)); - return component ? `${component.name}_${appSpec.name}` : null; +function anchorComponentIdentifier(appSpec) { + return gComponentIdentifiers(appSpec)[0] ?? null; } /** True iff the component's primary mount is any sync mount (g:/r:/s:). */ @@ -574,7 +604,9 @@ module.exports = { hasFlag, getComponentSyncMode, isGComponent, - electedComponentIdentifier, + gComponentIdentifiers, + syncedComponentIdentifiers, + anchorComponentIdentifier, isSyncedComponent, validateMountPath, validateSubdirOrFilename, diff --git a/ZelBack/src/services/utils/untilSucceeds.js b/ZelBack/src/services/utils/untilSucceeds.js new file mode 100644 index 0000000000..2d81681dc1 --- /dev/null +++ b/ZelBack/src/services/utils/untilSucceeds.js @@ -0,0 +1,28 @@ +const log = require('../../lib/log'); +const serviceHelper = require('../serviceHelper'); + +// How long a boot step waits before it runs again after a failure. +const BOOT_STEP_RETRY_MS = 15 * 1000; + +/** + * Runs `step` until it succeeds, waiting BOOT_STEP_RETRY_MS after each + * failure. Boot holds at a database step that fails rather than starting again + * from the top, so nothing boot already started runs twice. + * @param {string} what Names the step in the log. + * @param {function(): Promise<*>} step + * @returns {Promise<*>} What the step resolved. + */ +async function untilSucceeds(what, step) { + for (;;) { + try { + // eslint-disable-next-line no-await-in-loop + return await step(); + } catch (error) { + log.error(`${what} failed, running it again in ${BOOT_STEP_RETRY_MS / 1000}s: ${error.message}`); + // eslint-disable-next-line no-await-in-loop + await serviceHelper.delay(BOOT_STEP_RETRY_MS); + } + } +} + +module.exports = { untilSucceeds, BOOT_STEP_RETRY_MS }; diff --git a/ZelBack/src/services/utils/withinDeadline.js b/ZelBack/src/services/utils/withinDeadline.js new file mode 100644 index 0000000000..d82185e422 --- /dev/null +++ b/ZelBack/src/services/utils/withinDeadline.js @@ -0,0 +1,24 @@ +/** What withinDeadline answers when the deadline passed first. */ +const TIMED_OUT = 'timedOut'; + +/** + * What `work` resolved, or TIMED_OUT once `ms` pass. The work itself keeps + * running: this only stops the caller waiting for it. + * @param {Promise<*>} work + * @param {number} ms + * @returns {Promise<*>} + */ +async function withinDeadline(work, ms) { + let timer; + const deadline = new Promise((resolve) => { timer = setTimeout(resolve, ms, TIMED_OUT); }); + try { + return await Promise.race([work, deadline]); + } finally { + clearTimeout(timer); + } +} + +module.exports = { + TIMED_OUT, + withinDeadline, +}; diff --git a/helpers/hashes.json b/helpers/hashes.json index 8b80b18bfc..e2995661bb 100644 --- a/helpers/hashes.json +++ b/helpers/hashes.json @@ -13,5 +13,7 @@ "5bc27303e232f8bcbf1f531987e28f0a", "16379f485d76083906f1fd19790dd1c4", "adcaa6996078beaff5d62fe95b2774d2", - "b76e4ee20664bc6db7a91276f98f66b7" + "b76e4ee20664bc6db7a91276f98f66b7", + "0840c013921a322079de7d694417ff42", + "b9bfb6578c581b007f92137826c01cb1" ] diff --git a/package.json b/package.json index bbb2cff7b6..5ca8589a73 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "flux", - "version": "8.21.1", + "version": "8.22.0", "description": "Flux, Your Gateway to a Decentralized World", "repository": { "type": "git", @@ -79,7 +79,7 @@ "@aws-sdk/client-ecr": "^3.922.0", "@azure/identity": "^4.13.0", "@isaacs/ttlcache": "~1.4.1", - "@runonflux/nat-upnp": "~1.0.2", + "@runonflux/nat-upnp": "~2.0.5", "apicache": "~1.6.3", "archiver": "~7.0.1", "axios": "~1.13.6", diff --git a/test-infra/Dockerfile.fluxos b/test-infra/Dockerfile.fluxos index 35a763b76e..a4b445d0e3 100644 --- a/test-infra/Dockerfile.fluxos +++ b/test-infra/Dockerfile.fluxos @@ -48,6 +48,7 @@ RUN set -eu; \ conntrack \ tcpdump \ e2fsprogs \ + dmsetup \ systemd \ dbus \ openssh-client \ @@ -177,8 +178,12 @@ RUN set -eu; \ WORKDIR /flux -COPY package.json ./ -RUN npm install --production \ +# node_modules belongs to the account that owns the install, as it does on an +# operator's node: pm2 starts FluxOS through npm start, which runs npm install as +# that account. Installed as it, so no chown copies the tree into a layer of its own. +RUN chown fluxuser:fluxuser /flux +COPY --chown=fluxuser:fluxuser package.json ./ +RUN runuser -u fluxuser -- npm install --production \ && cp package-lock.json /tmp/image-package-lock.json # pm2, for a legacy node started the way the multitool starts one, at the @@ -203,9 +208,9 @@ RUN PM2_VERSION="$(tr -d '[:space:]' < /tmp/pm2-version)" \ # key it generates at first boot. Carried by the copy rather than a chown of its # own: on an overlay filesystem a recursive chown copies every file it touches up # into the container's writable layer, which on this tree is minutes of boot and -# reads as a node that never came up. node_modules keeps root's ownership - it is -# .dockerignore'd here and installed above, and FluxOS only reads it. Root ignores -# all of it, so the image is unchanged for a node that stays root. +# reads as a node that never came up. node_modules is .dockerignore'd here and +# installed above. Root ignores all of it, so the image is unchanged for a node that +# stays root. COPY --chown=fluxuser:fluxuser . . COPY --chown=fluxuser:fluxuser test-infra/fixtures/userconfig-template.js /flux/config/userconfig.js @@ -214,7 +219,7 @@ COPY --chown=fluxuser:fluxuser test-infra/fixtures/userconfig-template.js /flux/ # may carry another (development does not track one), and a node started by pm2 # runs npm install at every start with no registry to reach: it succeeds only # when the lockfile and node_modules agree. -RUN cp /tmp/image-package-lock.json /flux/package-lock.json +RUN cp /tmp/image-package-lock.json /flux/package-lock.json && chown fluxuser:fluxuser /flux/package-lock.json RUN touch /flux/debug.log /flux/error.log /flux/info.log /flux/warn.log \ && chown fluxuser:fluxuser /flux /flux/debug.log /flux/error.log /flux/info.log /flux/warn.log \ diff --git a/test-infra/build-release-image.sh b/test-infra/build-release-image.sh new file mode 100755 index 0000000000..1f36f3909f --- /dev/null +++ b/test-infra/build-release-image.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Builds a node image whose FluxOS is another ref's product code inside this tree's +# harness - the stubs, entrypoint and runner a fleet of this tree boots with - so a +# suite can boot a node on a released FluxOS and upgrade it in place to this tree's. +# +# FLUX_E2E_TAG= ./test-infra/build-release-image.sh +# +# The ref must be in this clone. Builds fluxos-01 only, stamped like any other +# (build-images.sh), from a temporary worktree of HEAD with the product paths +# taken from the ref and the per-node configs generated from the ref's own config. +# The gitignored build products the image needs are copied from this clone. +set -euo pipefail + +REF=${1:?usage: FLUX_E2E_TAG= $0 } +: "${FLUX_E2E_TAG:?FLUX_E2E_TAG names the tag to build under}" +CLONE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PRODUCT=(ZelBack helpers lib app.js apiServer.js init.js package.json) + +git -C "$CLONE" rev-parse --verify --quiet "$REF^{commit}" >/dev/null || { echo "build-release-image: $REF is not in this clone" >&2; exit 2; } + +WORK=$(mktemp -d "${TMPDIR:-/tmp}/flux-release-image.XXXXXX") +cleanup() { + git -C "$CLONE" worktree remove --force "$WORK/tree" >/dev/null 2>&1 || true + rm -rf "$WORK" +} +trap cleanup EXIT + +git -C "$CLONE" worktree add --quiet --detach "$WORK/tree" HEAD +cd "$WORK/tree" + +for p in "${PRODUCT[@]}"; do + rm -rf "$p" + git archive "$REF" -- "$p" | tar -x +done + +# Whatever the harness built and ignores: vendored libraries, fixture binaries. +git -C "$CLONE" ls-files --others --ignored --exclude-standard -- test-infra \ + | grep -v -e '/node_modules/' -e '^test-infra/runner/' \ + | while read -r f; do mkdir -p "$(dirname "$f")"; cp -p "$CLONE/$f" "$f"; done + +# The per-node configs a node of the ref's code reads, from the ref's production config. +mkdir -p config +cp test-infra/fixtures/userconfig-template.js config/userconfig.js +node test-infra/config/generate-configs.js +rm -f config/userconfig.js + +./test-infra/build-images.sh fluxos-01 +echo "build-release-image: flux-e2e-fluxos-01:$FLUX_E2E_TAG holds $REF ($(git -C "$CLONE" rev-parse --short "$REF^{commit}"))" diff --git a/test-infra/config/HARNESS_TUNABLES.md b/test-infra/config/HARNESS_TUNABLES.md index b3a116a520..6019ecaa1e 100644 --- a/test-infra/config/HARNESS_TUNABLES.md +++ b/test-infra/config/HARNESS_TUNABLES.md @@ -128,7 +128,7 @@ loses the property, so the factor is pinned by a unit test. | `fluxapps.portRestoreIntervalMs` | 600000 | 30000 | 20.0x | | `fluxapps.portTestBindDelayMs` | 5000 | 100 | 50.0x | | `fluxapps.portTestMaxAttempts` | 5 | 2 | 2.5x | -| `fluxapps.portTestPeerTimeoutMs` | 30000 | 3000 | 10.0x | +| `fluxapps.portTestPeerTimeoutMs` | 30000 | 8000 | 3.8x | | `fluxapps.portTestPropagationDelayMs` | 10000 | 100 | 100.0x | | `fluxapps.redeploy.composedDelay` | 5 | 1 | 5.0x | | `fluxapps.removal.delay` | 300 | 5 | 60.0x | diff --git a/test-infra/config/shared.js b/test-infra/config/shared.js index ab8ca5166c..c66064f785 100644 --- a/test-infra/config/shared.js +++ b/test-infra/config/shared.js @@ -12,7 +12,11 @@ module.exports = { // the behaviour is unchanged, only the searching stops. // nodeIp stays empty as in production: it is the node's own address for a port mapping, // it cannot be a shared constant, and no mapping is ever made because verification fails. - upnp: { gatewayUrl: 'http://31.200.0.6:3000/upnp/device.xml', nodeIp: '' }, + // A removal the router refuses is asked again after 2s, the wait doubling to 8s, so a + // suite sees several asks in seconds rather than the half hour production allows. + upnp: { + gatewayUrl: 'http://31.200.0.6:3000/upnp/device.xml', nodeIp: '', removalRetryFirstMs: 2000, removalRetryMaxMs: 8000, + }, // Empty disables analytics. The app default is the live cloudaudit endpoint, which // the fleet must never report suite activity to. analytics: { url: '' }, @@ -221,7 +225,10 @@ module.exports = { installCollisionWaitMs: 5000, portTestBindDelayMs: 100, portTestPropagationDelayMs: 100, - portTestPeerTimeoutMs: 3000, + // A peer answers only once its UDP probe of a silent port has reached its deadline + // (UDP_PROBE_DEADLINE_MS, 3 s, fluxNetworkHelper.js), so the wait for that peer has to + // outlast it, or every UDP refusal reads as a peer that never answered. + portTestPeerTimeoutMs: 8000, portTestMaxAttempts: 2, spawnReconfirmDelayMs: 30000, globalCmdDelayMs: 100, diff --git a/test-infra/entrypoint.sh b/test-infra/entrypoint.sh index 71e3d7b9f6..161da4c7c5 100755 --- a/test-infra/entrypoint.sh +++ b/test-infra/entrypoint.sh @@ -52,6 +52,34 @@ mkdir -p /dat/var/lib/fluxd \ /dat/usr/lib/fluxwatchdog \ /mnt/appdata/flux-apps +# Mount a filesystem image through a device-mapper device of its own, so a suite +# can freeze it (every write held) or suspend the device (every read and write +# held), framework/host-stall.js. Device-mapper names are the host kernel's, so +# each carries the node's hostname. The device and its loop are released lazily: +# both go away with the last mount of them, when the node's container does. +mount_stallable() { + local img="$1" mnt="$2" name loop + mountpoint -q "$mnt" && return 0 + name="flux-$(hostname)-$(basename "$mnt")" + loop="$(losetup -f --show "$img")" + dmsetup create --noudevsync "$name" --table "0 $(blockdev --getsz "$loop") linear $loop 0" + dmsetup mknodes "$name" + mount "/dev/mapper/$name" "$mnt" + dmsetup remove --deferred "$name" + losetup -d "$loop" +} + +# A node whose docker data a suite stalls (framework/host-stall.js) keeps it on a +# filesystem of its own, mounted before docker starts. Holding it holds docker's +# threads in the kernel, which is what a stalled host does to docker. The image +# lives on the node's volume, so a restarted node mounts the same data again. +if [ "$FLUX_FREEZABLE_DOCKER_ROOT" = "true" ]; then + img=/mnt/appdata/docker-root.img + [ -f "$img" ] || { truncate -s 8G "$img" && mkfs.ext4 -q -F "$img"; } + mkdir -p /mnt/appdata/docker + mount_stallable "$img" /mnt/appdata/docker +fi + # In stub mode FluxOS only needs somewhere to read an API key from; the calls # themselves go to the shared stub. In binary mode the config is syncthing's own # and this fixture must not be in the way of it. @@ -59,38 +87,9 @@ if [ "$FLUX_SYNCTHING_MODE" != "binary" ]; then cp /flux/test-infra/fixtures/syncthing-config.xml /dat/usr/lib/syncthing/config.xml 2>/dev/null || true fi -# Overlay test config into ZelBack/config/ so app.js loads it naturally. -# app.js hardcodes NODE_CONFIG_DIR to ZelBack/config/ (cannot be overridden -# from env — fluxbenchd hashes that directory for tamper detection). -if [ -n "$NODE_CONFIG_DIR" ] && [ -d "$NODE_CONFIG_DIR" ]; then - cp "$NODE_CONFIG_DIR"/default.js /flux/ZelBack/config/local.js - cp "$(dirname "$NODE_CONFIG_DIR")/shared.js" /flux/ZelBack/ 2>/dev/null || true -fi +# The node's config overlay: its per-node file with the runner's overrides over it. +/flux/test-infra/write-node-config.sh -# The runner's own overrides arrive as JSON and are merged OVER the per-node file -# copied above, which is where the per-node database names come from - replacing -# that file rather than merging would take them with it. -# -# They used to arrive as NODE_CONFIG. The config package merges that variable over -# every file whatever directory is pinned, so it could redirect any endpoint -# without touching the directory fluxbenchd hashes - the one change tamper -# detection cannot see. The entry points delete it now, and this carries the same -# content to the same place through a file instead. -if [ -n "$FLUX_TEST_CONFIG" ]; then - node -e ' - const fs = require("fs"); - const target = "/flux/ZelBack/config/local.js"; - const base = fs.existsSync(target) ? require(target) : {}; - const isPlain = (v) => v && typeof v === "object" && !Array.isArray(v); - const merge = (a, b) => { - const out = { ...a }; - for (const [k, v] of Object.entries(b)) out[k] = isPlain(v) && isPlain(a[k]) ? merge(a[k], v) : v; - return out; - }; - const merged = merge(base, JSON.parse(process.env.FLUX_TEST_CONFIG)); - fs.writeFileSync(target, `module.exports = ${JSON.stringify(merged, null, 2)};\n`); - ' -fi # A node whose discovery the harness has started rejoins the mesh by itself on # every later start, the run's own discoveryAutostart notwithstanding. node /flux/test-infra/rejoin-discovery.cjs apply @@ -212,7 +211,7 @@ if [ -f /sys/fs/cgroup/cgroup.controllers ]; then fi # Syncthing listens on apiport+2 in production. The availability checker tests -# that port. In systemd mode the stub forward runs as a unit instead +# that port. Against the stub, the forward to it runs as a unit # (syncthing-forward.service), so systemd supervises it like everything else. SYNCTHING_LISTEN_PORT=$((${FLUX_API_PORT:-16127} + 2)) if [ "$FLUX_SYNCTHING_MODE" = "binary" ]; then @@ -243,8 +242,6 @@ if [ "$FLUX_SYNCTHING_MODE" = "binary" ]; then # the flags a real Arcane node is supervised with, read off a live one /flux/test-infra/start-syncthing.sh fi -elif [ -n "$FLUX_SYNCTHING_HOST" ] && [ "$FLUX_SYSTEMD_MODE" != "true" ]; then - socat TCP-LISTEN:${SYNCTHING_LISTEN_PORT},fork,reuseaddr TCP:${FLUX_SYNCTHING_HOST}:${FLUX_SYNCTHING_PORT:-8384} & fi # Trust test registry CA for dockerd (Node.js uses NODE_EXTRA_CA_CERTS directly). @@ -263,231 +260,129 @@ if [ -n "$FLUX_BOOT_ID" ]; then echo "$FLUX_BOOT_ID" > /tmp/flux-boot-id fi -# ── systemd mode (opt-in) ────────────────────────────────────────────────── -# The node runs a real systemd as PID 1: dockerd and fluxos become units, as -# they are on a production host, and anything FluxOS manages through systemctl -# behaves as it does there. Everything below this block is the default path -# and is unreachable in this mode; the fault-injection lever that lives there -# (/tmp/fluxos.pid — restartFluxos) does not exist under systemd, and -# framework/systemd-control.js holds the equivalents. -if [ "$FLUX_SYSTEMD_MODE" = "true" ]; then - # fluxos.service runs FluxOS as root. An unprivileged node is a different - # install shape, and booting it as root would test the wrong one. - if [ -n "$FLUX_FLUXOS_USER" ]; then - echo "[entrypoint] FATAL: systemd mode runs FluxOS as root; FLUX_FLUXOS_USER=$FLUX_FLUXOS_USER is not supported here" >&2 - exit 1 - fi +# ── systemd ──────────────────────────────────────────────────────────────── +# The node runs a real systemd as PID 1, as a production host does: dockerd and +# FluxOS are units, anything FluxOS manages through systemctl behaves as it does +# there, and Docker places containers through systemd's cgroup driver. - # The Ubuntu base image's policy-rc.d refuses every service start a package - # install asks for. A host has none, so a package installed here starts its - # service as it does on a node. - rm -f /usr/sbin/policy-rc.d +# The Ubuntu base image's policy-rc.d refuses every service start a package +# install asks for. A host has none, so a package installed here starts its +# service as it does on a node. +rm -f /usr/sbin/policy-rc.d - # Container env does not cross into systemd services (the manager - # environment arrives empty), so dump it for the - # units' EnvironmentFile. node writes C-style-quoted values, which keeps - # NODE_CONFIG's embedded JSON quoting intact. - node -e ' - const fs = require("fs"); - const skip = new Set(["PATH", "HOSTNAME", "HOME", "PWD", "OLDPWD", "SHLVL", "TERM", "SHELL", "_", "DEBIAN_FRONTEND", "LS_COLORS"]); - const lines = Object.entries(process.env) - .filter(([k]) => !skip.has(k)) - .map(([k, v]) => `${k}=${JSON.stringify(v)}`); - fs.writeFileSync("/etc/fluxos-harness.env", lines.join("\n") + "\n"); - ' +# Container env does not cross into systemd services (the manager +# environment arrives empty), so dump it for the +# units' EnvironmentFile. node writes C-style-quoted values, which keeps +# NODE_CONFIG's embedded JSON quoting intact. +node -e ' + const fs = require("fs"); + const skip = new Set(["PATH", "HOSTNAME", "HOME", "PWD", "OLDPWD", "SHLVL", "TERM", "SHELL", "_", "DEBIAN_FRONTEND", "LS_COLORS"]); + const lines = Object.entries(process.env) + .filter(([k]) => !skip.has(k)) + .map(([k, v]) => `${k}=${JSON.stringify(v)}`); + fs.writeFileSync("/etc/fluxos-harness.env", lines.join("\n") + "\n"); +' - cp /flux/test-infra/systemd/*.service /etc/systemd/system/ - mkdir -p /etc/systemd/system/multi-user.target.wants - ln -sf /etc/systemd/system/dockerd.service /etc/systemd/system/multi-user.target.wants/dockerd.service +cp /flux/test-infra/systemd/*.service /etc/systemd/system/ +mkdir -p /etc/systemd/system/multi-user.target.wants +for unit in dockerd flux-node-network; do + ln -sf "/etc/systemd/system/$unit.service" "/etc/systemd/system/multi-user.target.wants/$unit.service" +done +# A legacy node the multitool installed runs FluxOS under pm2 as the operator's +# account (pm2-fluxuser.service); every other node runs the fluxos unit. +if [ "$FLUX_PM2" = "1" ]; then + for unit in pm2-fluxuser flux-pm2-logs; do + ln -sf "/etc/systemd/system/$unit.service" "/etc/systemd/system/multi-user.target.wants/$unit.service" + done +else ln -sf /etc/systemd/system/fluxos.service /etc/systemd/system/multi-user.target.wants/fluxos.service - if [ -n "$FLUX_SYNCTHING_HOST" ]; then - ln -sf /etc/systemd/system/syncthing-forward.service /etc/systemd/system/multi-user.target.wants/syncthing-forward.service - fi +fi - # Container hygiene: kernel modules cannot be loaded here, and a tmpfs - # over /tmp would shadow the harness's /tmp/flux-boot-config bind mount. - # kernel.*, fs.*, vm.* and binfmt_misc are the host's, not the container's, - # and a privileged node can write them: systemd-sysctl and systemd-binfmt - # would apply the image's settings to the shared runner host. - ln -sf /dev/null /etc/systemd/system/systemd-modules-load.service - ln -sf /dev/null /etc/systemd/system/systemd-sysctl.service - ln -sf /dev/null /etc/systemd/system/systemd-binfmt.service - ln -sf /dev/null /etc/systemd/system/tmp.mount +# The node's output: dockerd's, FluxOS's and the network setup's lines go to the +# container's own stream, where the harness reads them, through this FIFO. Its +# reader is started here so it holds the container's stdout; opened read-write, it +# never sees an end of file when a writer stops. +mkdir -p /var/lib/flux-harness +rm -f /var/lib/flux-harness/stream +mkfifo /var/lib/flux-harness/stream +cat 0<>/var/lib/flux-harness/stream & +# An unprivileged node runs FluxOS as the operator's account, as an operator's own +# install does; systemd gives the service that account's HOME, from which a legacy +# node derives its flux directory. The apps folder is the account's, and nothing +# else changes hands. +if [ -n "$FLUX_FLUXOS_USER" ]; then + mkdir -p /etc/systemd/system/fluxos.service.d + printf '[Service]\nUser=%s\n' "$FLUX_FLUXOS_USER" > /etc/systemd/system/fluxos.service.d/user.conf + chown "$(id -u "$FLUX_FLUXOS_USER"):$(id -g "$FLUX_FLUXOS_USER")" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" +fi +# A legacy node's FluxOS starts syncthing outside its own process tree, and pm2, +# which runs FluxOS on a legacy node, signals only that tree when FluxOS stops or +# dies: syncthing runs on. The fluxos unit signals FluxOS's own process alone. +if [ -z "$FLUXOS_PATH" ] && [ "$FLUX_PM2" != "1" ]; then + mkdir -p /etc/systemd/system/fluxos.service.d + printf '[Service]\nKillMode=process\n' > /etc/systemd/system/fluxos.service.d/legacy.conf +fi +if [ -n "$FLUX_SYNCTHING_HOST" ]; then + ln -sf /etc/systemd/system/syncthing-forward.service /etc/systemd/system/multi-user.target.wants/syncthing-forward.service +fi - # docker-ce's packaged containerd.service would boot under systemd and - # dockerd then prefers it — with its snapshotter on the overlayfs root - # (EINVAL on overlay-on-overlay). Masked, dockerd spawns its own - # containerd under the data-root, exactly like the default mode. - ln -sf /dev/null /etc/systemd/system/containerd.service +# Container hygiene: kernel modules cannot be loaded here, and a tmpfs +# over /tmp would shadow the harness's /tmp/flux-boot-config bind mount. +# kernel.*, fs.*, vm.* and binfmt_misc are the host's, not the container's, +# and a privileged node can write them: systemd-sysctl and systemd-binfmt +# would apply the image's settings to the shared runner host. +ln -sf /dev/null /etc/systemd/system/systemd-modules-load.service +ln -sf /dev/null /etc/systemd/system/systemd-sysctl.service +ln -sf /dev/null /etc/systemd/system/systemd-binfmt.service +ln -sf /dev/null /etc/systemd/system/tmp.mount - # docker-ce's own docker.service and docker.socket start a second dockerd on - # the same daemon.json. It takes the volume store's lock, then waits for the - # masked containerd, and dockerd.service can never open the store. Masked, - # dockerd.service is the node's only daemon. - ln -sf /dev/null /etc/systemd/system/docker.service - ln -sf /dev/null /etc/systemd/system/docker.socket +# docker-ce's packaged containerd.service would boot under systemd and +# dockerd then prefers it — with its snapshotter on the overlayfs root +# (EINVAL on overlay-on-overlay). Masked, dockerd spawns its own +# containerd under the data-root. +ln -sf /dev/null /etc/systemd/system/containerd.service - # The default mode's data-root, in the file the dockerd unit reads its - # configuration from. - mkdir -p /etc/docker - cat > /etc/docker/daemon.json < /etc/docker/daemon.json <{}); w.close();' 2>/dev/null; then - echo "[entrypoint] FATAL: cannot allocate an inotify instance — systemd will exit 255." >&2 - echo "[entrypoint] fs.inotify.max_user_instances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null) is a HOST-WIDE per-uid pool" >&2 - echo "[entrypoint] shared by every container on this box. Raise it on the HOST, not here." >&2 - exit 3 - fi - echo "[entrypoint] setup complete, handing off to systemd as pid 1" - # journal-or-kmsg, not the default: systemd's own messages go to the journal once - # journald exists, and to the KERNEL RING BUFFER before it does — which is the exact - # window a boot failure happens in. The default target leaves that window writing to - # /dev/console, and a container without a TTY has none, so a systemd that dies before - # journald is completely mute. The fallback only fires when the journal is - # unavailable, so a healthy boot writes nothing to kmsg. - exec /lib/systemd/systemd --log-target=journal-or-kmsg -fi - -# Start dockerd under a tiny watchdog so it is respawned if it exits. Production -# nodes run dockerd under systemd (which restarts it); this mirrors that and lets -# tests bounce dockerd (kill it) to exercise the reconciler's reconnect/orphan -# recovery without bricking the node. node app.js stays PID 1 (via exec below). -rm -f /var/run/docker.pid -( - set +e - while true; do - rm -f /var/run/docker.pid - dockerd --data-root /mnt/appdata/docker - echo "dockerd exited (rc=$?), respawning in 1s" >&2 - sleep 1 - done -) & - -TIMEOUT=30 -ELAPSED=0 -until docker info > /dev/null 2>&1; do - if [ "$ELAPSED" -ge "$TIMEOUT" ]; then - echo "ERROR: dockerd failed to start within ${TIMEOUT}s" >&2 - exit 1 - fi - sleep 1 - ELAPSED=$((ELAPSED + 1)) -done -echo "dockerd is ready (took ${ELAPSED}s)" - -# A named network shape (test-infra/network-shapes.sh), declared per node by the -# suite, built before FluxOS starts because a node reads its network at boot. -# Built after dockerd is up, as a VPN comes up after docker on a host: dockerd -# refuses to start when every private range it could give its bridge overlaps a -# route, and the def1 split routes overlap them all. Not swallowed, for the -# reason given for the default route: a shape that failed to build would leave a -# plain static node answering for it. -if [ -n "${FLUX_E2E_NETWORK_SHAPE:-}" ]; then - if ! /flux/test-infra/network-shapes.sh "$FLUX_E2E_NETWORK_SHAPE"; then - echo "ERROR: could not build network shape $FLUX_E2E_NETWORK_SHAPE" >&2 - exit 1 - fi -fi - -# A global IPv6 address, declared per node by the suite (createTestEnv globalIpv6). It sits on -# a device of its own with no IPv6 route beyond it, so the node holds a routable IPv6 address -# as a dual-stack host does while every IPv6 connection fails at once (ENETUNREACH): an IPv6 -# that is configured and does not route. Independent of the network shape, and built before -# FluxOS starts for the same reason. -if [ -n "${FLUX_E2E_GLOBAL_IPV6:-}" ]; then - if ! { ip link add flux6 type dummy \ - && sysctl -qw net.ipv6.conf.flux6.disable_ipv6=0 \ - && ip link set flux6 up \ - && ip -6 addr add "$FLUX_E2E_GLOBAL_IPV6/64" dev flux6 nodad; }; then - echo "ERROR: could not add the global IPv6 address $FLUX_E2E_GLOBAL_IPV6" >&2 - exit 1 - fi -fi - -# WHO FLUXOS RUNS AS. Root unless the fleet names an account, which is the Arcane -# node; named, it is the unprivileged install - FluxOS as the user the operator -# installed it as, with passwordless sudo for everything privileged. +# The handoff marker. Everything above runs as PID 1's shell, whose output IS the +# container's stdout; after it, the units' lines reach that stdout only through the +# stream FIFO, and systemd's own output does not reach it at all. +# So a container that died with this line in `docker logs` failed under systemd (read +# its journal), and one that died WITHOUT it failed in the setup above, where the +# failing command's own stderr is the diagnosis. +# systemd's FIRST act is to create an inotify instance to watch cgroups, and +# fs.inotify.max_user_instances is a PER-UID, HOST-WIDE pool that every container +# on the box draws from as root. Exhaust it and systemd cannot allocate its manager +# object and PID 1 exits 255 — having written the reason to /dev/console, which a +# container without a TTY does not have. That is total silence: no docker logs, no +# journal (journald never started), nothing. # -# What that account owns is decided here rather than in the image, because it is -# per-node: the tree FluxOS writes into, its logs, and the apps folder, which on an -# unprivileged node belongs to the operator. Nothing else changes hands - dockerd, -# syncthing and every app volume stay root's, and that is what makes the node's own -# reads refusable, exactly as a container's mount point is on a real node. -# -# setpriv rather than sudo: this node's environment IS its configuration, and sudo -# rebuilds the environment it passes on. HOME travels with the account because a -# legacy node derives its flux directory from it. -if [ -n "$FLUX_FLUXOS_USER" ]; then - FLUXOS_UID="$(id -u "$FLUX_FLUXOS_USER")" - FLUXOS_GID="$(id -g "$FLUX_FLUXOS_USER")" - HOME="$(getent passwd "$FLUX_FLUXOS_USER" | cut -d: -f6)" - export HOME - # The apps folder only. The install itself already belongs to this account, from - # the image - a recursive chown of it here would copy the whole tree up into the - # overlay and outlast the node's readiness window. - chown "$FLUXOS_UID:$FLUXOS_GID" "${FLUX_APPS_FOLDER:-/mnt/appdata/flux-apps}" - set -- setpriv --reuid="$FLUXOS_UID" --regid="$FLUXOS_GID" --init-groups "$@" +# So ask the question here, where the answer can be printed. node is in this image; +# fs.watch allocates exactly the resource systemd is about to need. +if ! node -e 'const w=require("fs").watch("/tmp",()=>{}); w.close();' 2>/dev/null; then + echo "[entrypoint] FATAL: cannot allocate an inotify instance — systemd will exit 255." >&2 + echo "[entrypoint] fs.inotify.max_user_instances=$(cat /proc/sys/fs/inotify/max_user_instances 2>/dev/null) is a HOST-WIDE per-uid pool" >&2 + echo "[entrypoint] shared by every container on this box. Raise it on the HOST, not here." >&2 + exit 3 fi - -# Run FluxOS (CMD ["node","app.js"]) under a respawn watchdog instead of exec'ing it -# as PID 1. This mirrors the dockerd watchdog above and production's systemd: the -# entrypoint shell stays PID 1 and node runs as a child, so a test can kill+respawn -# the FluxOS process (restartFluxos) WITHOUT restarting the container or the inner -# dockerd - the app containers keep running, exactly like `systemctl restart fluxos`. -# The child PID is written to /tmp/fluxos.pid so a test kills only the node process, -# never PID 1. A SIGTERM/SIGINT (docker stop at teardown) stops the child and exits. -# -# While /tmp/fluxos.hold exists the loop does not respawn: the node stays down -# after FluxOS exits, as a machine does between a shutdown and its next boot, -# and comes back once a test removes the file. -# A legacy node the multitool installed: the pm2 daemon starts FluxOS through -# start.sh, which runs npm start - npm install, then FluxOS - from the account's -# home, with the flags the multitool gives it. The kill timeout is the fleet's -# to choose; unset, pm2 uses its own default. There is no registry here, so npm -# answers its install from the image's own. The entrypoint stays in the -# foreground streaming pm2's logs, so FluxOS's output is the container's, and a -# test stops and starts FluxOS through pm2 as an operator or the OS does. -if [ "$FLUX_PM2" = "1" ]; then - export NPM_CONFIG_OFFLINE=true NPM_CONFIG_AUDIT=false NPM_CONFIG_FUND=false NPM_CONFIG_UPDATE_NOTIFIER=false - cd "$HOME" || exit 1 - pm2 start "$HOME/zelflux/start.sh" --name flux \ - --max-memory-restart 1500M --restart-delay 30000 --max-restarts 40 --time \ - ${FLUX_PM2_KILL_TIMEOUT_MS:+--kill-timeout "$FLUX_PM2_KILL_TIMEOUT_MS"} >/dev/null - trap 'pm2 kill >/dev/null 2>&1; exit 0' TERM INT - pm2 logs --raw --lines 0 & - wait $! - exit 0 -fi - -set +e -STOPPING=0 -trap 'STOPPING=1; kill -TERM "$(cat /tmp/fluxos.pid 2>/dev/null)" 2>/dev/null' TERM INT -while [ "$STOPPING" = "0" ]; do - "$@" & - FLUXOS_PID=$! - echo "$FLUXOS_PID" > /tmp/fluxos.pid - wait "$FLUXOS_PID" - [ "$STOPPING" = "1" ] && break - while [ -f /tmp/fluxos.hold ] && [ "$STOPPING" = "0" ]; do sleep 1; done - [ "$STOPPING" = "1" ] && break - echo "fluxos (node app.js) exited, respawning in 1s" >&2 - sleep 1 -done +echo "[entrypoint] setup complete, handing off to systemd as pid 1" +# journal-or-kmsg, not the default: systemd's own messages go to the journal once +# journald exists, and to the KERNEL RING BUFFER before it does — which is the exact +# window a boot failure happens in. The default target leaves that window writing to +# /dev/console, and a container without a TTY has none, so a systemd that dies before +# journald is completely mute. The fallback only fires when the journal is +# unavailable, so a healthy boot writes nothing to kmsg. +exec /lib/systemd/systemd --log-target=journal-or-kmsg diff --git a/test-infra/external-http-stub/index.js b/test-infra/external-http-stub/index.js index 4a3776424e..dfb67583a2 100644 --- a/test-infra/external-http-stub/index.js +++ b/test-infra/external-http-stub/index.js @@ -392,7 +392,8 @@ function imageSyncthingVersion() { const state = { // Nodes behind a UPnP gateway, by fleet address: the external address it reports (the - // node's own when null), its port mappings, and every action the node sent it. + // node's own when null), how many mappings its table holds (no limit when null), its port + // mappings, and every action the node sent it. upnpGateways: {}, // Bytes served INSTEAD of the signed artifact, under the signed artifact's own name. A // mirror or CDN answering correctly-named requests with the wrong content, which is the @@ -407,6 +408,11 @@ const state = { // enterprise node, which is what every suite that is not about enterprise placement wants. // Absent would be a different thing entirely - see the policy block below. enterpriseNodes: {}, + // Whether a port closed over UDP counts toward the availability check's DOS. null is + // the document absent from the published bundle, which is the default. + udpPortDos: null, + // Whether the pre-install refusal over UDP is off. null is the document absent. + udpInstallRefusalOff: null, latestRelease: { tag_name: 'v0.0.0', name: 'stub-release' }, geolocation: {}, // The syncthing the node image ships, read from the repository the image was built @@ -542,6 +548,8 @@ function resignPolicy({ bumpSeq = true } = {}) { vettedrepositories: state.vettedRepositories, tamperingblockednodes: state.tamperingBlocklist, enterprisenodes: state.enterpriseNodes, + ...(state.udpPortDos === null ? {} : { udpportdos: state.udpPortDos }), + ...(state.udpInstallRefusalOff === null ? {} : { udpinstallrefusaloff: state.udpInstallRefusalOff }), }, // Content-addressed, as the real publisher does it: the plain name is mutable and means // "the current table", so the bundle names the hash a fetch should produce. Derived from @@ -715,8 +723,10 @@ app.get('/repos/:owner/:repo', (req, res) => { // fails on it, and the node runs without UPnP as it does on a network with none. // // A node behind a gateway is served one: a WANIPConnection service whose control URL answers -// the four actions upnpService sends - the external address, and adding, listing and -// deleting port mappings - from that node's own mapping table. +// the actions upnpService sends - the external address, and adding, looking up, listing and +// deleting port mappings - from that node's own mapping table. Its table refuses what a +// router's does: a port another machine holds (718), and a new entry once it holds as many as +// its capacity (728). const UPNP_SERVICE = 'urn:schemas-upnp-org:service:WANIPConnection:1'; function upnpGatewayOf(req) { @@ -765,31 +775,87 @@ app.post('/upnp/control', express.text({ type: '*/*' }), (req, res) => { const action = String(req.get('SOAPAction') || '').replace(/"/g, '').split('#')[1] || ''; const arg = (name) => (String(req.body).match(new RegExp(`<${name}>([^<]*)`)) || [])[1] ?? ''; const sameEntry = (m) => m.externalPort === Number(arg('NewExternalPort')) && m.protocol === arg('NewProtocol').toUpperCase(); - gateway.requests.push({ action, at: new Date().toISOString() }); + gateway.requests.push({ + action, + port: Number(arg('NewExternalPort')) || null, + protocol: arg('NewProtocol').toUpperCase() || null, + description: arg('NewPortMappingDescription') || null, + at: new Date().toISOString(), + }); switch (action) { case 'GetExternalIPAddress': soapReply(res, action, { NewExternalIPAddress: gateway.externalIp ?? clientAddress(req) }); return; case 'AddPortMapping': { + const held = gateway.mappings.find(sameEntry); + if (held && held.internalClient !== arg('NewInternalClient')) { + soapFault(res, 718, 'ConflictInMappingEntry'); + return; + } + if (!held && gateway.capacity !== null && gateway.mappings.length >= gateway.capacity) { + gateway.requests[gateway.requests.length - 1].refused = 728; + soapFault(res, 728, 'NoPortMapsAvailable'); + return; + } + const protocolFault = gateway.protocolFaults?.[arg('NewProtocol').toUpperCase()]; + if (protocolFault) { + gateway.requests[gateway.requests.length - 1].refused = protocolFault; + soapFault(res, protocolFault, 'ActionFailed'); + return; + } gateway.mappings = gateway.mappings.filter((m) => !sameEntry(m)); gateway.mappings.push({ externalPort: Number(arg('NewExternalPort')), protocol: arg('NewProtocol').toUpperCase(), internalPort: Number(arg('NewInternalPort')), internalClient: arg('NewInternalClient'), - description: arg('NewPortMappingDescription'), + // a FRITZ!Box stores a description with its underscores and spaces as dots + description: gateway.fritzDescriptions ? arg('NewPortMappingDescription').replace(/[_ ]/g, '.') : arg('NewPortMappingDescription'), leaseDuration: Number(arg('NewLeaseDuration')), }); soapReply(res, action, {}); return; } case 'DeletePortMapping': { + if (gateway.refuseDeletes > 0) { + gateway.refuseDeletes -= 1; + gateway.requests[gateway.requests.length - 1].refused = 501; + soapFault(res, 501, 'ActionFailed'); + return; + } const before = gateway.mappings.length; gateway.mappings = gateway.mappings.filter((m) => !sameEntry(m)); + // a router that removes the entry and answers with a fault, and answers a + // removal of one it does not hold with the same fault, never 714 + if (gateway.deleteFault) { + gateway.requests[gateway.requests.length - 1].refused = gateway.deleteFault; + soapFault(res, gateway.deleteFault, 'ActionFailed'); + return; + } if (gateway.mappings.length === before) soapFault(res, 714, 'NoSuchEntryInArray'); else soapReply(res, action, {}); return; } + case 'GetSpecificPortMappingEntry': { + if (gateway.lookupFault) { + gateway.requests[gateway.requests.length - 1].refused = gateway.lookupFault; + soapFault(res, gateway.lookupFault, 'ActionFailed'); + return; + } + const entry = gateway.mappings.find(sameEntry); + if (!entry) { + soapFault(res, 714, 'NoSuchEntryInArray'); + return; + } + soapReply(res, action, { + NewInternalPort: entry.internalPort, + NewInternalClient: entry.internalClient, + NewEnabled: 1, + NewPortMappingDescription: entry.description, + NewLeaseDuration: entry.leaseDuration, + }); + return; + } case 'GetGenericPortMappingEntry': { const entry = gateway.mappings[Number(arg('NewPortMappingIndex'))]; if (!entry) { @@ -984,12 +1050,84 @@ control.post('/storage-requests/reset', (req, res) => { // Put a node behind a UPnP gateway, with no mappings yet. control.post('/upnp-gateway', (req, res) => { - const { ip, externalIp = null } = req.body || {}; + const { + ip, externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, fritzDescriptions = false, + } = req.body || {}; if (!ip) { res.status(400).json({ error: 'ip is required' }); return; } - state.upnpGateways[ip] = { externalIp, mappings: [], requests: [] }; + state.upnpGateways[ip] = { + externalIp, capacity, mappings, refuseDeletes, fritzDescriptions, requests: [], + }; + res.json({ ok: true }); +}); + +// Set how many DeletePortMapping requests a node's gateway refuses from now on, +// keeping its table and the requests it has seen. +control.post('/upnp-gateway/:ip/refuse-deletes', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.refuseDeletes = Number(req.body?.count ?? 0); + res.json({ ok: true }); +}); + +// Set the fault a node's gateway answers every lookup with from now on, whether +// or not it holds the entry (null: answer as the spec says), keeping its table +// and the requests it has seen. +control.post('/upnp-gateway/:ip/lookup-fault', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.lookupFault = req.body?.code ?? null; + res.json({ ok: true }); +}); + +// Set the fault a node's gateway answers every removal with from now on, while +// still carrying it out (null: answer as the spec says), keeping its table and +// the requests it has seen. +control.post('/upnp-gateway/:ip/delete-fault', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.deleteFault = req.body?.code ?? null; + res.json({ ok: true }); +}); + +// Set the fault a node's gateway answers every mapping of one protocol with from +// now on (null: maps it as the spec says), keeping its table and the requests it +// has seen - a router that forwards TCP and refuses UDP. +control.post('/upnp-gateway/:ip/protocol-fault', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + const protocol = String(req.body?.protocol ?? '').toUpperCase(); + if (!['TCP', 'UDP'].includes(protocol)) { + res.status(400).json({ error: 'protocol must be TCP or UDP' }); + return; + } + gateway.protocolFaults = { ...gateway.protocolFaults, [protocol]: req.body?.code ?? null }; + res.json({ ok: true }); +}); + +// Set how many mappings a node's gateway holds from now on (null: no limit), +// keeping its table and the requests it has seen. +control.post('/upnp-gateway/:ip/capacity', (req, res) => { + const gateway = state.upnpGateways[req.params.ip]; + if (!gateway) { + res.status(404).json({ error: 'no gateway for that ip' }); + return; + } + gateway.capacity = req.body?.capacity ?? null; res.json({ ok: true }); }); @@ -1160,6 +1298,8 @@ control.post('/policy', (req, res) => { vettedrepositories: 'vettedRepositories', tamperingblockednodes: 'tamperingBlocklist', enterprisenodes: 'enterpriseNodes', + udpportdos: 'udpPortDos', + udpinstallrefusaloff: 'udpInstallRefusalOff', }; for (const [name, value] of Object.entries(body.documents)) { if (!map[name]) return res.status(400).json({ error: `unknown document ${name}` }); @@ -1207,6 +1347,8 @@ control.post('/reset', (req, res) => { state.vettedRepositories = []; state.tamperingBlocklist = []; state.enterpriseNodes = {}; + state.udpPortDos = null; + state.udpInstallRefusalOff = null; state.latestRelease = { tag_name: 'v0.0.0', name: 'stub-release' }; state.geolocation = {}; artifacts.clear(); diff --git a/test-infra/network-shapes.sh b/test-infra/network-shapes.sh index 489beb22ee..9babc7cb9b 100755 --- a/test-infra/network-shapes.sh +++ b/test-infra/network-shapes.sh @@ -50,6 +50,19 @@ # direct-nameserver the node's resolv.conf names the fleet's resolver itself, # as a host that lists a public resolver does, rather than # Docker's embedded server in front of it. +# udp-blocked no UDP datagram starting a flow reaches the node on its +# fleet device, as behind a router that forwards TCP alone; +# replies to the node's own datagrams still arrive, and +# loopback (Docker's embedded DNS) is untouched. +# udp-first-lost the first UDP datagram from each source starting a flow on +# the fleet device is dropped, and every later one within +# 30 s arrives: loss that a probe sending once never survives +# and one that sends again always does. +# udp-every-other every other UDP datagram starting a flow on the fleet +# device is dropped, whatever its source: steady loss of half. +# udp-replies-lost no reply to a UDP datagram the node sent arrives on its fleet +# device: a peer that probes a port and never hears back, +# whatever the port does. set -euo pipefail shape="$1" @@ -156,6 +169,19 @@ case "$shape" in : "${FLUX_E2E_RESOLVER:?}" printf 'nameserver %s\n' "$FLUX_E2E_RESOLVER" > /etc/resolv.conf ;; + udp-blocked) + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -j DROP + ;; + udp-replies-lost) + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate ESTABLISHED -j DROP + ;; + udp-every-other) + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m statistic --mode nth --every 2 --packet 0 -j DROP + ;; + udp-first-lost) + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m recent --name udpseen --rcheck --seconds 30 -j ACCEPT + iptables -A INPUT -i "$dev" -p udp -m conntrack --ctstate NEW -m recent --name udpseen --set -j DROP + ;; *) echo "network-shapes: unknown shape '$shape'" >&2 exit 1 diff --git a/test-infra/node-network.sh b/test-infra/node-network.sh new file mode 100755 index 0000000000..2b8d124b87 --- /dev/null +++ b/test-infra/node-network.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# The node's network as its suite declares it, built once dockerd is up and before +# FluxOS starts (flux-node-network.service), because a node reads its network at boot. +# +# A named network shape (test-infra/network-shapes.sh) is built after dockerd is up, +# as a VPN comes up after docker on a host: dockerd refuses to start when every +# private range it could give its bridge overlaps a route, and the def1 split routes +# overlap them all. Not swallowed: a shape that failed to build would leave a plain +# static node answering for it. +set -e +if [ -n "${FLUX_E2E_NETWORK_SHAPE:-}" ]; then + if ! /flux/test-infra/network-shapes.sh "$FLUX_E2E_NETWORK_SHAPE"; then + echo "ERROR: could not build network shape $FLUX_E2E_NETWORK_SHAPE" >&2 + exit 1 + fi +fi + +# A global IPv6 address, declared per node by the suite (createTestEnv globalIpv6). It +# sits on a device of its own with no IPv6 route beyond it, so the node holds a +# routable IPv6 address as a dual-stack host does while every IPv6 connection fails at +# once (ENETUNREACH): an IPv6 that is configured and does not route. Independent of the +# network shape. +if [ -n "${FLUX_E2E_GLOBAL_IPV6:-}" ]; then + if ! { ip link add flux6 type dummy \ + && sysctl -qw net.ipv6.conf.flux6.disable_ipv6=0 \ + && ip link set flux6 up \ + && ip -6 addr add "$FLUX_E2E_GLOBAL_IPV6/64" dev flux6 nodad; }; then + echo "ERROR: could not add the global IPv6 address $FLUX_E2E_GLOBAL_IPV6" >&2 + exit 1 + fi +fi diff --git a/test-infra/runner/framework/container.js b/test-infra/runner/framework/container.js index c809013f43..5e00d6d335 100644 --- a/test-infra/runner/framework/container.js +++ b/test-infra/runner/framework/container.js @@ -228,7 +228,7 @@ export async function getAppContainerExitCode(container, appName, componentName) /** * Bounce the inner dockerd under a running FluxOS (the dockerd-restart orphan - * case). Kills dockerd; the in-image watchdog respawns it. Without --live-restore + * case). Kills dockerd; systemd restarts it (dockerd.service). Without --live-restore * this stops dockerd's containers, leaving them 'exited' for the reconnect sweep * to recover. Confirms dockerd actually went DOWN and came back UP, so the caller * can't observe a false "already ready". @@ -251,19 +251,38 @@ export async function restartDockerd(container, { readyTimeoutMs = 40000, interv throw new Error(`restartDockerd: dockerd did not cycle down and back up within ${readyTimeoutMs}ms`); } +// The node's FluxOS runs as the fluxos unit (test-infra/systemd/fluxos.service), which +// systemd restarts whenever its process exits. A held node is one whose unit carries a +// runtime drop-in turning the restart off: its FluxOS stays down after it exits, as a +// machine does between a shutdown and its next boot, until releaseFluxos. +const FLUXOS_HOLD = '/run/systemd/system/fluxos.service.d/hold.conf'; + +/** + * The pid of the node's FluxOS process, or null while it is not running. + * @returns {Promise} + */ +export async function fluxosPid(container) { + const pid = Number((await execInContainer(container, 'systemctl show -p MainPID --value fluxos')).stdout.trim()); + return pid > 0 ? pid : null; +} + +async function holdFluxos(container) { + const r = await execInContainer(container, + `mkdir -p "$(dirname ${FLUXOS_HOLD})" && printf '[Service]\nRestart=no\n' > ${FLUXOS_HOLD} && systemctl daemon-reload`); + if (r.exitCode !== 0) throw new Error(`holdFluxos: ${r.output}`); +} + /** * Restart the FluxOS process only - the `systemctl restart fluxos` case. Kills just - * the node app.js child (its PID is in /tmp/fluxos.pid, written by the entrypoint - * watchdog, so PID 1 is never touched); the watchdog respawns it. The inner dockerd - * and the running app containers are NOT affected - they keep running while FluxOS's - * in-memory state (e.g. controllerDesired) is wiped. This is distinct from - * restartNode (whole container -> dockerd + containers restart) and restartDockerd - * (dockerd only). Confirms FluxOS went DOWN and came back UP so the caller can't - * observe a false "already ready". + * FluxOS; systemd starts it again. The inner dockerd and the running app containers + * are NOT affected - they keep running while FluxOS's in-memory state (e.g. + * controllerDesired) is wiped. This is distinct from restartNode (whole container -> + * dockerd + containers restart) and restartDockerd (dockerd only). Confirms FluxOS + * went DOWN and came back UP so the caller can't observe a false "already ready". */ export async function restartFluxos(container, { apiPort = 16127, readyTimeoutMs = 120000, interval = 500 } = {}) { - // hard-kill only the node child (state wiped instantly); never PID 1 - await execInContainer(container, 'kill -9 "$(cat /tmp/fluxos.pid 2>/dev/null)" 2>/dev/null || true'); + // hard-kill only FluxOS (state wiped instantly) + await execInContainer(container, 'systemctl kill --kill-whom=main -s KILL fluxos'); const probe = `curl -sf -o /dev/null http://127.0.0.1:${apiPort}/flux/version`; const start = Date.now(); let sawDown = false; @@ -313,9 +332,10 @@ export async function getContainerImageDigest(container, appName, componentName) export async function shutdownFluxosGracefully(container, { hold = false, stopSyncthingAfter = false, exitTimeoutMs = 120000, interval = 250, } = {}) { - const pid = Number((await execInContainer(container, 'cat /tmp/fluxos.pid')).stdout.trim()); - if (!pid) throw new Error('shutdownFluxosGracefully: no FluxOS pid in /tmp/fluxos.pid'); - await execInContainer(container, `touch /run/nologin${hold ? ' /tmp/fluxos.hold' : ''}`); + const pid = await fluxosPid(container); + if (!pid) throw new Error('shutdownFluxosGracefully: FluxOS is not running'); + if (hold) await holdFluxos(container); + await execInContainer(container, 'touch /run/nologin'); if (stopSyncthingAfter) { // Inside the node, so the daemon stops within a poll of the exit rather than // a docker exec round trip later. @@ -360,7 +380,9 @@ export async function shutdownFluxosGracefully(container, { // Bring back a node held down by shutdownFluxosGracefully({ hold: true }), and // wait for its API to answer. export async function releaseFluxos(container, { apiPort = 16127, readyTimeoutMs = 120000, interval = 500 } = {}) { - await execInContainer(container, 'rm -f /tmp/fluxos.hold'); + const released = await execInContainer(container, + `rm -f ${FLUXOS_HOLD} && systemctl daemon-reload && systemctl reset-failed fluxos; systemctl start fluxos`); + if (released.exitCode !== 0) throw new Error(`releaseFluxos: ${released.output}`); const probe = `curl -sf -o /dev/null http://127.0.0.1:${apiPort}/flux/version`; const start = Date.now(); while (Date.now() - start < readyTimeoutMs) { @@ -418,9 +440,9 @@ export async function configuredIpOf(container) { // shutdown handling at all. `hold` keeps the node down afterwards, as a machine // that lost power stays down; releaseFluxos brings it back. export async function crashFluxos(container, { hold = false, exitTimeoutMs = 30000, interval = 250 } = {}) { - const pid = Number((await execInContainer(container, 'cat /tmp/fluxos.pid')).stdout.trim()); - if (!pid) throw new Error('crashFluxos: no FluxOS pid in /tmp/fluxos.pid'); - if (hold) await execInContainer(container, 'touch /tmp/fluxos.hold'); + const pid = await fluxosPid(container); + if (!pid) throw new Error('crashFluxos: FluxOS is not running'); + if (hold) await holdFluxos(container); await execInContainer(container, `kill -9 ${pid}`); const start = Date.now(); while (Date.now() - start < exitTimeoutMs) { @@ -433,17 +455,22 @@ export async function crashFluxos(container, { hold = false, exitTimeoutMs = 300 throw new Error(`crashFluxos: FluxOS ${pid} still running ${exitTimeoutMs}ms after SIGKILL`); } +// A command as a legacy node's operator runs pm2: as their account, against their pm2 +// home (test-infra/systemd/pm2-fluxuser.service). +export const AS_PM2_OPERATOR = 'runuser -u fluxuser -- env PM2_HOME=/home/fluxuser/.pm2'; +const PM2 = `${AS_PM2_OPERATOR} pm2`; + // The kill timeout of FluxOS's pm2 registration on a legacy node whose FluxOS // pm2 runs (createTestEnv pm2Nodes), as the daemon holds it and as `pm2 save` // stored it for the next boot. null where pm2 has none, which is its default. // // @returns {Promise<{live: number|null, saved: number|null, restarts: number}>} export async function pm2Registration(container) { - const jlist = await execInContainer(container, 'pm2 jlist'); + const jlist = await execInContainer(container, `${PM2} jlist`); if (jlist.exitCode !== 0) throw new Error(`pm2Registration: pm2 jlist failed: ${jlist.output}`); const live = JSON.parse(jlist.stdout.trim()).find((p) => p.name === 'flux'); if (!live) throw new Error('pm2Registration: pm2 lists no process named flux'); - const dump = await execInContainer(container, 'cat "$HOME/.pm2/dump.pm2" 2>/dev/null || echo "[]"'); + const dump = await execInContainer(container, 'cat /home/fluxuser/.pm2/dump.pm2 2>/dev/null || echo "[]"'); const saved = JSON.parse(dump.stdout.trim()).find((p) => p.name === 'flux'); return { live: live.pm2_env.kill_timeout ?? null, @@ -464,7 +491,7 @@ export async function shutdownFluxosUnderPm2(container, { stopSyncthingAfter = f await execInContainer(container, 'touch /run/nologin'); try { const started = Date.now(); - const stopped = await execInContainer(container, 'pm2 stop flux'); + const stopped = await execInContainer(container, `${PM2} stop flux`); const stopMs = Date.now() - started; if (stopped.exitCode !== 0) throw new Error(`shutdownFluxosUnderPm2: pm2 stop failed: ${stopped.output}`); if (stopSyncthingAfter) await execInContainer(container, 'pkill -KILL -x syncthing; true'); @@ -477,7 +504,7 @@ export async function shutdownFluxosUnderPm2(container, { stopSyncthingAfter = f // Start FluxOS again through pm2 on a node shut down by shutdownFluxosUnderPm2, // and wait for its API to answer. export async function startFluxosUnderPm2(container, { apiPort = 16127, readyTimeoutMs = 180000, interval = 1000 } = {}) { - const started = await execInContainer(container, 'pm2 start flux'); + const started = await execInContainer(container, `${PM2} start flux`); if (started.exitCode !== 0) throw new Error(`startFluxosUnderPm2: pm2 start failed: ${started.output}`); const probe = `curl -sf -o /dev/null http://127.0.0.1:${apiPort}/flux/version`; const start = Date.now(); diff --git a/test-infra/runner/framework/db-client.js b/test-infra/runner/framework/db-client.js index 0c0b6bce67..f76f224cfb 100644 --- a/test-infra/runner/framework/db-client.js +++ b/test-infra/runner/framework/db-client.js @@ -64,6 +64,13 @@ export function dbClient(nodeNum) { await localDb.collection('ownaddress').deleteMany({}); }, + // The node keeps no record of when it was last alive, as on a node whose FluxOS + // never wrote one: its next boot reads as a first boot. + async forgetHeartbeat() { + const localDb = await db('local'); + await localDb.collection('nodestartuptracker').deleteOne({ _id: 'heartbeat' }); + }, + // The node's own geolocation record: what it decided about its address. async nodeGeolocation() { const localDb = await db('local'); @@ -111,8 +118,27 @@ export function dbClient(nodeNum) { return localDb.collection('zelappsinformation').countDocuments({}); }, + // The names of the apps this node holds, from its own table: the route that + // lists them is cached, and this is not. + async localAppNames() { + const localDb = await db('appsLocal'); + const rows = await localDb.collection('zelappsinformation').find({}, { projection: { _id: 0, name: 1 } }).toArray(); + return rows.map((row) => row.name); + }, + // This node's view of where an app runs (the rows fluxapprunning messages // upsert and the empty-broadcast bug used to wipe). + // This node's rows for the app at that address stay unexpired for `ms` from now, + // as a production record does across a restart the harness's compressed record + // would not outlive. Resolves to how many rows were found. + async extendAppLocation(appName, ip, ms) { + const globalDb = await db('appsGlobal'); + const host = String(ip).replace(/:\d+$/, '').replace(/\./g, '\\.'); + const { matchedCount } = await globalDb.collection('zelappslocation') + .updateMany({ name: appName, ip: { $regex: `^${host}(:\\d+)?$` } }, { $set: { expireAt: new Date(Date.now() + ms) } }); + return matchedCount; + }, + async getAppLocations(appName) { const globalDb = await db('appsGlobal'); return globalDb.collection('zelappslocation').find({ name: appName }).toArray(); @@ -123,6 +149,16 @@ export function dbClient(nodeNum) { return globalDb.collection('zelappslocation').find({ ip }).toArray(); }, + // A location's expiry on this node: its rows for the app at that address are + // gone, as the TTL index removes them. Resolves to how many rows went. + async expireAppLocation(appName, ip) { + const globalDb = await db('appsGlobal'); + const host = String(ip).replace(/:\d+$/, '').replace(/\./g, '\\.'); + const { deletedCount } = await globalDb.collection('zelappslocation') + .deleteMany({ name: appName, ip: { $regex: `^${host}(:\\d+)?$` } }); + return deletedCount; + }, + // The restart attempts this node's reconciler has recorded for a component: // the burst window and the backoff ladder, as many entries as each holds. async restartAttempts(identifier) { @@ -424,6 +460,45 @@ export function dbClient(nodeNum) { await localDb.collection('zelappsinformation').insertOne(spec); }, + // The app as this node has it installed. + async localApp(name) { + const localDb = await db('appsLocal'); + return localDb.collection('zelappsinformation').findOne({ name }, { projection: { _id: 0 } }); + }, + + // When this node took each app it holds: { name: placedAt }. + async placements() { + const localDb = await db('appsLocal'); + const records = await localDb.collection('zelappsplacements').find({}, { projection: { _id: 0 } }).toArray(); + return Object.fromEntries(records.map(({ name, placedAt }) => [name, placedAt])); + }, + + async seedPlacement(name, placedAt) { + const localDb = await db('appsLocal'); + await localDb.collection('zelappsplacements').insertOne({ name, placedAt: new Date(placedAt) }); + }, + + async deletePlacement(name) { + const localDb = await db('appsLocal'); + await localDb.collection('zelappsplacements').deleteMany({ name }); + }, + + // This node's location row for one app at one address, or null. + async appLocationAt(name, ip) { + const globalDb = await db('appsGlobal'); + return globalDb.collection('zelappslocation').findOne({ name, ip }, { projection: { _id: 0 } }); + }, + + async deleteAppLocation(name, ip) { + const globalDb = await db('appsGlobal'); + await globalDb.collection('zelappslocation').deleteMany({ name, ip }); + }, + + async setAppLocationRunningSince(name, ip, runningSince) { + const globalDb = await db('appsGlobal'); + await globalDb.collection('zelappslocation').updateOne({ name, ip }, { $set: { runningSince: new Date(runningSince) } }); + }, + async seedInstallingLocation({ name, ip, broadcastedAt }) { const globalDb = await db('appsGlobal'); diff --git a/test-infra/runner/framework/external-http-control.js b/test-infra/runner/framework/external-http-control.js index aaa7c23749..d63d04db6f 100644 --- a/test-infra/runner/framework/external-http-control.js +++ b/test-infra/runner/framework/external-http-control.js @@ -191,15 +191,86 @@ export async function resetStorageRequests() { * Put a node behind a UPnP gateway the stub serves it. Every other node is served * a gateway with no WAN service, and runs without UPnP. * @param {string} ip - the node's fleet address, as subnet-config names it - * @param {{externalIp?: string}} [opts] - the address the gateway reports; the node's own when absent + * @param {object} [opts] + * @param {string} [opts.externalIp] - the address the gateway reports; the node's own when absent + * @param {number} [opts.capacity] - how many mappings its table holds; no limit when absent + * @param {object[]} [opts.mappings] - what its table holds already, as upnpGatewayState lists it + * @param {number} [opts.refuseDeletes] - how many DeletePortMapping requests it refuses with + * 501 before it removes anything; a refused request carries `refused: 501` in its state + * @param {boolean} [opts.fritzDescriptions] - store each new mapping's description as a + * FRITZ!Box does, its underscores and spaces as dots */ -export async function enableUpnpGateway(ip, { externalIp = null } = {}) { - const res = await post('/upnp-gateway', { ip, externalIp }); +export async function enableUpnpGateway(ip, { + externalIp = null, capacity = null, mappings = [], refuseDeletes = 0, fritzDescriptions = false, +} = {}) { + const res = await post('/upnp-gateway', { + ip, externalIp, capacity, mappings, refuseDeletes, fritzDescriptions, + }); if (!res.ok) throw new Error(`enableUpnpGateway: ${JSON.stringify(res)}`); } /** - * The gateway a node is behind: its port mappings and the actions the node sent it. + * Set how many DeletePortMapping requests a node's gateway refuses from now on, keeping its + * table and the requests it has seen. + * @param {string} ip + * @param {number} count - Infinity refuses every one + */ +export async function setUpnpRefuseDeletes(ip, count) { + const res = await post(`/upnp-gateway/${ip}/refuse-deletes`, { count: Number.isFinite(count) ? count : Number.MAX_SAFE_INTEGER }); + if (!res.ok) throw new Error(`setUpnpRefuseDeletes: ${JSON.stringify(res)}`); +} + +/** + * Set the fault a node's gateway answers every mapping of one protocol with from now on - a + * router that forwards TCP and refuses UDP. Keeps its table and the requests it has seen; each + * such answer carries `refused: `. + * @param {string} ip + * @param {'TCP'|'UDP'} protocol + * @param {number|null} code - null maps the protocol as the spec says + */ +export async function setUpnpProtocolFault(ip, protocol, code) { + const res = await post(`/upnp-gateway/${ip}/protocol-fault`, { protocol, code }); + if (!res.ok) throw new Error(`setUpnpProtocolFault: ${JSON.stringify(res)}`); +} + +/** + * Set the fault a node's gateway answers every GetSpecificPortMappingEntry with from now on, + * whether or not it holds the entry. Keeps its table and the requests it has seen; each such + * answer carries `refused: `. + * @param {string} ip + * @param {number|null} code - null answers as the spec says + */ +export async function setUpnpLookupFault(ip, code) { + const res = await post(`/upnp-gateway/${ip}/lookup-fault`, { code }); + if (!res.ok) throw new Error(`setUpnpLookupFault: ${JSON.stringify(res)}`); +} + +/** + * Set the fault a node's gateway answers every removal with from now on, while still carrying + * it out, and the same fault for a removal of an entry it does not hold, where the spec says + * 714. Keeps its table and the requests it has seen; each such answer carries `refused: `. + * @param {string} ip + * @param {number|null} code - null answers as the spec says + */ +export async function setUpnpDeleteFault(ip, code) { + const res = await post(`/upnp-gateway/${ip}/delete-fault`, { code }); + if (!res.ok) throw new Error(`setUpnpDeleteFault: ${JSON.stringify(res)}`); +} + +/** + * Set how many mappings a node's gateway holds from now on, keeping its table and the requests + * it has seen. A new mapping past it is refused with 728, carried as `refused: 728` in its state. + * @param {string} ip + * @param {number|null} capacity - null for no limit + */ +export async function setUpnpCapacity(ip, capacity) { + const res = await post(`/upnp-gateway/${ip}/capacity`, { capacity }); + if (!res.ok) throw new Error(`setUpnpCapacity: ${JSON.stringify(res)}`); +} + +/** + * The gateway a node is behind: its port mappings and the actions the node sent it, each with + * the external port, protocol and description it named (null for one it names none of). * @param {string} ip * @returns {Promise<{externalIp: string|null, mappings: object[], requests: object[]}|null>} */ diff --git a/test-infra/runner/framework/g-components.js b/test-infra/runner/framework/g-components.js new file mode 100644 index 0000000000..736366e44a --- /dev/null +++ b/test-infra/runner/framework/g-components.js @@ -0,0 +1,84 @@ +import { expect } from 'chai'; +import { listAppContainers } from './container.js'; +import { REGISTRY_REPO_HOST } from './subnet-config.js'; +import { sleepUnlessInfraDead } from './infra-death.js'; + +// An app with more than one single-writer (g:) component, as the suites that watch +// its election see it. +// +// The election decides the app, not each component: its primary runs every g: +// component on one node, and every other holder runs none. A node running one of +// them while another node runs any other is the app split across two nodes - the +// load balancer sends all of its ports to one of them, and the components cannot +// reach each other over the app's network - so every wait built from this holds +// that no two nodes run any of its g: components at once. + +/** + * One g: component of the test image, which writes `.txt` into its volume. + * Ports are left to buildSeedableApp, which allocates them by compose index; a + * component added by an update takes its own (allocatePortFor(appName, index)). + * @param {string} appName + * @param {string} name + * @returns {object} a compose entry + */ +export function gComponent(appName, name) { + return { + name, + description: `${name} of an app with several writers`, + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [`WRITE_FILE=/appdata/${name}.txt`, `WRITE_CONTENT=${name}`], + commands: [], + containerPorts: [80], + containerData: 'g:/appdata', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +/** + * What each holder runs of an app's g: components. + * @param {() => object} getEnv the suite's env, read when called + * @param {object} opts + * @param {string} opts.appName + * @param {string[]} opts.components every g: component name the app has at any + * point of the suite, in the order results list them + * @param {number[]} opts.holders node indices that hold the app + * @returns {{ + * folderOf: (name: string) => string, + * runningOn: (i: number) => Promise, + * oneHolderAtMost: () => Promise>, + * holdsFor: (ms: number, expected: Object) => Promise, + * }} + */ +export function watchGComponents(getEnv, { appName, components, holders }) { + const folderOf = (name) => `flux${name}_${appName}`; + const runningOn = async (i) => { + const containers = await listAppContainers(getEnv().clients[i].container); + return components.filter((name) => containers.some((c) => c.name === folderOf(name) && c.status?.startsWith('Up'))); + }; + // Asserted on every call: a waitHolding poll built on it fails at the first breach. + const oneHolderAtMost = async () => { + const running = await Promise.all(holders.map(runningOn)); + const runners = holders.filter((_, k) => running[k].length); + expect(runners.length, `the app runs on more than one node: ${holders.map((i, k) => `node ${i} runs [${running[k]}]`).join(', ')}`).to.be.at.most(1); + return Object.fromEntries(holders.map((i, k) => [i, running[k]])); + }; + // Held over a span of election passes, not read once. + const holdsFor = async (ms, expected) => { + const deadline = Date.now() + ms; + while (Date.now() < deadline) { + // eslint-disable-next-line no-await-in-loop + const running = await oneHolderAtMost(); + expect(running, 'what each holder runs').to.deep.equal(expected); + // eslint-disable-next-line no-await-in-loop + await sleepUnlessInfraDead(5000); + } + }; + return { + folderOf, runningOn, oneHolderAtMost, holdsFor, + }; +} diff --git a/test-infra/runner/framework/host-stall.js b/test-infra/runner/framework/host-stall.js new file mode 100644 index 0000000000..9a45b82e70 --- /dev/null +++ b/test-infra/runner/framework/host-stall.js @@ -0,0 +1,151 @@ +// A stalled host, on a real kernel: the filesystem a node's docker keeps its data on, +// held. Frozen (freezeDocker), every write docker makes there is held in the kernel - +// its threads sit in uninterruptible sleep and never run - until the filesystem is +// thawed, which is what a host that stops returning the disk does to docker. Stalled +// (stallDisk), every read is held as well, which is what a disk that stops answering +// does to everything on it. +// +// The node is booted with createTestEnv({ freezableDockerRoot: [index] }), which +// mounts its docker data root on a filesystem of its own, through a device-mapper +// device of its own. The env's teardown thaws and resumes it, so a node +// a failing test left held can still be stopped. +import { execInContainer } from './container.js'; + +export const DOCKER_ROOT = '/mnt/appdata/docker'; + +async function run(container, command, what) { + const result = await execInContainer(container, command); + if (result.exitCode !== 0) throw new Error(`${what}: ${(result.output || result.stderr || '').trim()}`); + return result.stdout; +} + +/** Hold every write the node's docker makes to its data root, until thawDocker. */ +export async function freezeDocker(container) { + await run(container, `mountpoint -q ${DOCKER_ROOT}`, `${DOCKER_ROOT} is not a mount of its own: boot the node with freezableDockerRoot`); + await run(container, `fsfreeze -f ${DOCKER_ROOT}`, 'freezeDocker'); +} + +/** Let the node's docker write again. A data root that is not frozen is left as it is. */ +export async function thawDocker(container) { + // fsfreeze refuses to thaw a filesystem that is not frozen, with EINVAL + await run(container, `out=$(fsfreeze -u ${DOCKER_ROOT} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }`, 'thawDocker'); +} + +// The device-mapper device a stallable mount sits on, in the node's shell. +const deviceOf = (mountpoint) => `$(findmnt -no SOURCE ${mountpoint} | sed -n 's|^/dev/mapper/||p')`; + +/** + * Hold every read and write to the filesystem at `mountpoint`, until resumeDisk: + * the filesystem is frozen, so a write is held before it reaches the cache, and + * its device suspended. What of it the kernel has cached is dropped in between, + * for that filesystem only, so a read of a file or a directory listing on it + * reaches the device and is held. + * @param {object} container The node. + * @param {string} mountpoint DOCKER_ROOT, mounted stallable. + */ +export async function stallDisk(container, mountpoint) { + await run(container, `d=${deviceOf(mountpoint)}; [ -n "$d" ]`, `${mountpoint} is not on a device of its own: boot the node with freezableDockerRoot`); + await run(container, `fsfreeze -f ${mountpoint} && find ${mountpoint} -xdev -type f -print0 | xargs -0 -r -n 64 -P 8 sh -c 'for f; do dd if="$f" iflag=nocache count=0 status=none; done' sh` + + ` && blockdev --flushbufs $(findmnt -no SOURCE ${mountpoint}) && dmsetup suspend --nolockfs --noflush ${deviceOf(mountpoint)}`, `stallDisk ${mountpoint}`); +} + +/** Let reads and writes reach the filesystem at `mountpoint` again. One not stalled is left as it is. */ +export async function resumeDisk(container, mountpoint) { + // the thaw writes to the device, so the device is resumed first + await run(container, `dmsetup resume ${deviceOf(mountpoint)} && { out=$(fsfreeze -u ${mountpoint} 2>&1) || echo "$out" | grep -q 'Invalid argument' || { echo "$out"; exit 1; }; }`, `resumeDisk ${mountpoint}`); +} + +/** + * Whether a listing of `dir` and a read of every file named `file` below it are + * still held after `overMs`: run in the background and left to finish once the + * disk is resumed, since a read held by the kernel cannot be killed. + * @returns {Promise} + */ +export async function readsHeld(container, dir, { file, overMs = 3000 } = {}) { + const marker = `/tmp/reads-returned-${Date.now()}`; + const read = `ls ${dir} >/dev/null${file ? ` && find ${dir} -name ${file} -exec cat {} + >/dev/null` : ''}`; + const result = await execInContainer(container, + `setsid sh -c '${read} && touch ${marker}' /dev/null 2>&1 & sleep ${overMs / 1000}; test ! -e ${marker}`); + return result.exitCode === 0; +} + +/** + * Thaw and resume every node's docker data root that is held, and every cgroup frozen in it + * (systemd-control.freezeUnit), and pass over the rest: the last step before + * anything reads a node through its docker or stops it, since a node whose docker + * is held can be neither read nor stopped. Never rejects. + * @param {Array<{container?: object}>} clients + */ +export async function thawEveryNode(clients) { + const thaw = `d=${deviceOf(DOCKER_ROOT)}; [ -n "$d" ] && dmsetup resume "$d" 2>/dev/null; ` + + `fsfreeze -u ${DOCKER_ROOT} 2>/dev/null; ` + + 'for f in $(find /sys/fs/cgroup -name cgroup.freeze 2>/dev/null); do [ "$(cat $f)" = 1 ] && echo 0 > $f; done; true'; + await Promise.all(clients.map(async (client) => { + if (!client?.container) return; + await execInContainer(client.container, thaw).catch(() => {}); + })); +} + +/** + * The cgroup directory a container's main process lives in, read while docker + * still answers. + * @param {object} container The node. + * @param {string} name The container's name inside the node. + * @returns {Promise} Under /sys/fs/cgroup. + */ +export async function containerCgroup(container, name) { + const pid = (await run(container, `docker inspect -f '{{.State.Pid}}' ${name}`, `finding ${name}'s process`)).trim(); + const line = (await run(container, `cat /proc/${pid}/cgroup`, `reading ${name}'s cgroup`)).trim(); + return line.replace(/^0::/, '/sys/fs/cgroup'); +} + +/** + * The processes in a cgroup and every cgroup below it that could still run: + * neither gone nor with SIGKILL pending. Read inside the node, independently of + * FluxOS. Throws when the cgroup does not exist, which is never an answer of + * "nothing can run". + * @param {object} container + * @param {string} cgroupDir Under /sys/fs/cgroup: systemd-control.unitCgroup, or containerCgroup. + * @returns {Promise} Their pids; empty when nothing in it can run again. + */ +export async function processesThatCanRun(container, cgroupDir) { + await run(container, `test -d '${cgroupDir}'`, `${cgroupDir} does not exist`); + const script = `for p in $(find '${cgroupDir}' -name cgroup.procs -exec cat {} +); do ` + + 's=/proc/$p/status; [ -r $s ] || continue; ' + + 'k=$(awk \'/^(SigPnd|ShdPnd):/{print $2}\' $s | while read m; do [ $(( 0x$m & 0x100 )) -ne 0 ] && echo y; done); ' + + '[ -z "$k" ] && echo $p; done; true'; + const out = await run(container, script, `reading the processes under ${cgroupDir}`); + return out.split('\n').map((line) => line.trim()).filter(Boolean); +} + +/** + * One reading of every thread of the processes named `comm` in the node: its + * state, its voluntary plus nonvoluntary context switches, and where it waits. + * @returns {Promise>} By thread id. + */ +async function readThreads(container, comm) { + const script = `for p in $(pidof ${comm}); do for t in /proc/$p/task/*; do ` + + 'echo "${t##*/} $(cut -d" " -f3 $t/stat) $(awk \'/ctxt_switches/{s+=$2} END{print s}\' $t/status) $(cat $t/wchan)"; ' + + 'done; done; true'; + const out = await run(container, script, `reading ${comm}'s threads`); + return new Map(out.split('\n').filter(Boolean).map((line) => { + const [tid, state, switches, wchan] = line.split(' '); + return [tid, { state, switches: Number(switches), wchan }]; + })); +} + +/** + * The threads of `comm` that did not run at all across `overMs`: in D at both + * readings, with the same switch count. What FluxOS's detector reads as stuck, + * read independently of it. + * @returns {Promise>} + */ +export async function stuckThreads(container, comm, { overMs = 3000 } = {}) { + const first = await readThreads(container, comm); + await new Promise((resolve) => { setTimeout(resolve, overMs); }); + const second = await readThreads(container, comm); + return [...second].filter(([tid, now]) => { + const before = first.get(tid); + return before && before.state === 'D' && now.state === 'D' && before.switches === now.switches; + }).map(([tid, now]) => ({ tid, wchan: now.wchan })); +} diff --git a/test-infra/runner/framework/log-on-failure.js b/test-infra/runner/framework/log-on-failure.js index 3537baee14..1da92221ed 100644 --- a/test-infra/runner/framework/log-on-failure.js +++ b/test-infra/runner/framework/log-on-failure.js @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { activeTestEnvs } from './test-env.js'; import { execInContainer } from './container.js'; import { SYNCTHING_HOME } from './syncthing-real.js'; +import { thawEveryNode } from './host-stall.js'; import failedHooksModule from './failed-hooks.cjs'; const { failedHooks } = failedHooksModule; @@ -155,12 +156,15 @@ export function dumpLogsOnFailure(getEnv) { // daemon, so they are fetched up front and written alongside them. Each entry // is best-effort (see env.infraDiagnostics) — a log fetch that failed carries // an `error` and is reported, never thrown. + // Everything below reads the nodes through their docker, which a suite may + // have left frozen. + await Promise.all(envs.map((env) => thawEveryNode(env.clients ?? []))); const infraByEnv = await Promise.all(envs.map((env) => env.infraDiagnostics())); const cgroupsByEnv = await Promise.all(envs.map((env) => cgroupState(env).catch( (err) => `cgroup probe failed: ${err.message}\n`, ))); - // A systemd-mode node's FluxOS writes to its journal, never to the container - // stream the log collectors read; see env.nodeJournals. + // What a node's units write to the journal rather than to the container stream + // the log collectors read; see env.nodeJournals. const journalsByEnv = await Promise.all(envs.map((env) => env.nodeJournals().catch( (err) => [{ index: 0, text: `journal read failed: ${err.message}\n` }], ))); diff --git a/test-infra/runner/framework/mongo-control.js b/test-infra/runner/framework/mongo-control.js new file mode 100644 index 0000000000..a7ca8b6d90 --- /dev/null +++ b/test-infra/runner/framework/mongo-control.js @@ -0,0 +1,90 @@ +import { MongoClient } from 'mongodb'; +import { getSubnetConfig } from './subnet-config.js'; + +// The fleet's mongod made to hold the commands FluxOS sends, while it goes on +// answering the driver's heartbeats: what a mongod with one thread stuck in the +// kernel looks like from FluxOS. Every data command waits and none fails, and +// the server still reads as healthy, so nothing but a bound on the call itself +// ends the wait. (A paused container answers no heartbeat either, and the driver +// then fails every call in flight by itself.) +// +// The harness mongod runs with enableTestCommands, which the failCommand +// failpoint needs. The harness's own database client (dbClient) is held with +// FluxOS, so a suite reads nothing through it meanwhile. + +const HELD_COMMANDS = [ + 'find', 'aggregate', 'getMore', 'count', 'distinct', + 'insert', 'update', 'delete', 'findAndModify', + 'createIndexes', 'listIndexes', 'dropIndexes', 'drop', 'listCollections', +]; + +async function withAdmin(work) { + const client = await MongoClient.connect(`mongodb://${getSubnetConfig().mongo}:27017`, { directConnection: true }); + try { + return await work(client.db('admin')); + } finally { + await client.close(); + } +} + +export async function holdCommands(holdMs) { + await withAdmin((admin) => admin.command({ + configureFailPoint: 'failCommand', + mode: 'alwaysOn', + data: { failCommands: HELD_COMMANDS, blockConnection: true, blockTimeMS: holdMs }, + })); +} + +// Turning the failpoint off waits for every command inside it to leave, and a held +// command stays for the whole hold. So the held commands are killed until the +// release completes: their wait is interruptible, and they leave at once. +// +// Resolves to how many commands were held from each client address (ip, no port): +// read on the server, so it says the calls reached mongod and waited, whatever the +// client did about it. +export async function releaseCommands() { + return withAdmin(async (admin) => { + const heldFrom = new Map(); + let released = false; + const off = admin.command({ configureFailPoint: 'failCommand', mode: 'off' }).finally(() => { released = true; }); + while (!released) { + // eslint-disable-next-line no-await-in-loop + const { inprog } = await admin.command({ currentOp: 1, active: true }); + const held = inprog.filter((op) => op.command && HELD_COMMANDS.some((name) => name in op.command)); + for (const op of held) heldFrom.set(op.opid, (op.client ?? '').replace(/:\d+$/, '')); + // eslint-disable-next-line no-await-in-loop + await Promise.all(held.map((op) => admin.command({ killOp: 1, op: op.opid }).catch(() => {}))); + // eslint-disable-next-line no-await-in-loop + await Promise.race([off.catch(() => {}), new Promise((resolve) => { setTimeout(resolve, 1000); })]); + } + await off; + const byClient = {}; + for (const ip of heldFrom.values()) byClient[ip] = (byClient[ip] ?? 0) + 1; + return byClient; + }); +} + +// The given commands on one collection of one node fail, as a call that ran out of time +// does (MaxTimeMSExpired): the driver neither retries it nor drops its other +// connections, so nothing but the named commands fails. Every other node's go through: +// each node keeps its data in databases of its own (dbClient's prefix). `times` fails +// that many and then stops; left out, every one fails until released. Resolves to a +// release that turns the failure off and resolves to how many commands it failed. +export async function failCommands(nodeNum, database, collection, { commands, times } = {}) { + const namespace = `node${String(nodeNum).padStart(2, '0')}_${database}.${collection}`; + await withAdmin((admin) => admin.command({ + configureFailPoint: 'failCommand', + mode: times === undefined ? 'alwaysOn' : { times }, + data: { failCommands: commands, namespace, errorCode: 50 }, + })); + return async function release() { + // turning a failpoint off answers how many times it fired + const { count } = await withAdmin((admin) => admin.command({ configureFailPoint: 'failCommand', mode: 'off' })); + return count; + }; +} + +// The next `times` writes to one collection of one node fail (failCommands). +export async function failWrites(nodeNum, database, collection, { times = 1 } = {}) { + return failCommands(nodeNum, database, collection, { commands: ['insert', 'update'], times }); +} diff --git a/test-infra/runner/framework/node-client.js b/test-infra/runner/framework/node-client.js index d90e705cb2..34af75674f 100644 --- a/test-infra/runner/framework/node-client.js +++ b/test-infra/runner/framework/node-client.js @@ -195,6 +195,10 @@ export function nodeClient(nodeNum) { 'masterSlave:operatorStartSettled', 'stream:gap', 'boot:settled', + // an app boot could not confirm this node still holds, held stopped; and the + // answer once its location record could be read + 'boot:appUnconfirmed', + 'boot:appConfirmed', 'confirmation:changed', 'daemon:polled', 'daemon:recovered', @@ -217,9 +221,18 @@ export function nodeClient(nodeNum) { // wording of the response stream. 'app:installOutcome', 'app:removed', + 'app:rebuilt', + 'app:softRedeployed', + 'reinstall:passEnded', 'app:componentRedeployed', 'app:specStored', 'app:running', + // what this node records of when it took each app it holds + 'app:placementRecorded', + 'app:placementRemoved', + 'app:placementsTidied', + 'app:placementsBackfilled', + 'app:announcementRefused', 'app:announcing', 'fileoperation:imageAcquired', 'fileoperation:imageDiscarded', @@ -231,6 +244,8 @@ export function nodeClient(nodeNum) { // it. Nothing existing carries that. 'ports:notOurs', 'ports:unproven', + 'ports:proven', + 'ports:udpUnreachable', 'discovery:started', 'peers:added', 'peers:accepting', @@ -280,6 +295,8 @@ export function nodeClient(nodeNum) { 'network:originatorNotListed', 'syncthing:namesVisible', 'syncthing:localChangesReverted', + 'syncthing:localChangesKept', + 'syncthing:localChangesChecked', 'syncthing:devicesResumed', 'pm2:killTimeoutRaiseFailed', 'pm2:killTimeoutUnchanged', @@ -302,7 +319,13 @@ export function nodeClient(nodeNum) { 'spawner:networkErrorSkip', 'spawner:paused', 'spawner:resumed', + 'placementHold:set', + 'placementHold:cleared', + 'hostStall:appUnloaded', + 'hostStall:appsStopped', 'network:apprunning', + // entries of an apprunning message this node refused, by gossip or sync + 'network:apprunningRefused', 'network:appinstalling', 'network:appinstallingerror', 'network:appremoved', @@ -317,6 +340,7 @@ export function nodeClient(nodeNum) { 'sync:refused', 'ephemeralSync:peerTimedOut', 'ephemeralSync:peerUnverified', + 'ephemeralSync:peerWriteFailed', 'ephemeralSync:peerDisconnected', 'ephemeralSync:budgetSpent', 'sync:chunkVerified', @@ -548,6 +572,20 @@ export function nodeClient(nodeNum) { const res = await controlFetch(`${url}/apps/installapplocally/${appname}`, { headers: { zelidauth } }); return res.text(); }, + // Test an app on THIS node: install it, check it starts, tear it down. By + // name, or by the hash of a temporary message for an app not yet paid for. + // Streams like installapplocally, so drained as text. + testAppInstall: async (appname, zelidauth) => { + const res = await controlFetch(`${url}/apps/testappinstall/${appname}`, { headers: { zelidauth } }); + return res.text(); + }, + // Remove an app from THIS node only. The third path segment is `global`, and + // true would remove the app from every node holding it, so it is always false + // here. Streams progress, so drained as text. + removeAppLocally: async (appname, zelidauth, { force = true } = {}) => { + const res = await controlFetch(`${url}/apps/appremove/${appname}/${force}/false`, { headers: { zelidauth } }); + return res.text(); + }, // Redeploy the WHOLE app on this node, reading the spec from the global table // rather than from what is installed - so a spec seeded since the install is // what the redeploy is judged against. Body is drained as text for the same diff --git a/test-infra/runner/framework/reconciler-suite.js b/test-infra/runner/framework/reconciler-suite.js index 8638c02811..891adf1320 100644 --- a/test-infra/runner/framework/reconciler-suite.js +++ b/test-infra/runner/framework/reconciler-suite.js @@ -120,6 +120,13 @@ export async function seedSpecUpdate(env, updated, indices) { // Returns the indices it installed on. export async function installOnNodes(env, app, indices, { timeout = 120000 } = {}) { await seedGlobalSpec(env, app, indices); + return installSeededOnNodes(env, app, indices, { timeout }); +} + +// installOnNodes for an app whose chain records the nodes already hold: an app +// uninstalled from a node and installed on it again, say. A local uninstall +// leaves those records, and seeding them a second time is refused. +export async function installSeededOnNodes(env, app, indices, { timeout = 120000 } = {}) { const teamKey = fluxTeamKey(); await Promise.all(indices.map(async (i) => { const client = env.clients[i]; @@ -221,12 +228,12 @@ export async function placeGAppInOrder(env, app, { const installAfter = env.clients[i].getLastEventId(); // eslint-disable-next-line no-await-in-loop await installOnNodes(env, app, [i]); - // Wait out the sync layer's first-run reset before writing disk data, or the - // reset deletes it - a folder pinned synced later must hold what its index claims. + // Every holder enters the election empty, as a newborn app's holders do: the + // sync layer's first-run reset has cleared the volume before the next holder is + // placed. A folder a suite pins synced later gets its bytes from the synced-data + // keeper. // eslint-disable-next-line no-await-in-loop await waitForReconcileActuated(env.clients[i], identifier, 'dataCleared', 60000, { afterId: installAfter }); - // eslint-disable-next-line no-await-in-loop - await seedSyncScopedData(env, app.spec.name, i); // WAITED FOR, not slept past. The ordering these placements exist to create // is the runningSince stamps, and a fixed gap produces it only while every @@ -553,9 +560,13 @@ export async function seedSyncthingApp(env, { // Seed the configurable test-app (controllable exit code / timed exit) and wait // for it to install. Returns { app, index, identifier }. Requires the test-app // binary to be built (bash test-infra/test-app/build.sh). -export async function seedTestApp(env, { name, exitCode = 0, exitAfterS = null } = {}) { +export async function seedTestApp(env, { + name, exitCode = 0, exitAfterS = null, logEveryMs = null, +} = {}) { await pushTestApp(name); - const app = await buildSeedableTestApp({ name, exitCode, exitAfterS }); + const app = await buildSeedableTestApp({ + name, exitCode, exitAfterS, logEveryMs, + }); const index = await seedAndInstall(env, app); return { app, index, identifier: `${name}_${name}` }; } diff --git a/test-infra/runner/framework/seed-helper.js b/test-infra/runner/framework/seed-helper.js index e487705cd4..e9a6f72880 100644 --- a/test-infra/runner/framework/seed-helper.js +++ b/test-infra/runner/framework/seed-helper.js @@ -259,7 +259,8 @@ export async function buildSeedableLegacyApp({ // whose chain starts later, and assertAliveOnThisChain refuses it. env = null, height = (env?.initialHeight ?? DEFAULT_INITIAL_HEIGHT) + 10, - instances = 1, + // below version 8 a specification asks for at least 3, as every one on the chain does + instances = 3, owner = null, expire = 22000, allowExternalRepotag = false, @@ -285,6 +286,8 @@ export async function buildSeedableLegacyApp({ cpu: 0.1, ram: 100, hdd: 1, + // every version 3 specification on the chain carries it, as false + tiered: false, instances, expire, }; @@ -394,7 +397,7 @@ export async function buildSeedableSyncthingApp({ * registry-helper.pushTestApp(name). */ export async function buildSeedableTestApp({ - name, exitCode = 0, exitAfterS = null, exitAfterMs = null, ...rest + name, exitCode = 0, exitAfterS = null, exitAfterMs = null, logEveryMs = null, ...rest }) { const environmentParameters = [`EXIT_CODE=${exitCode}`]; if (exitAfterS != null) environmentParameters.push(`EXIT_AFTER_S=${exitAfterS}`); @@ -402,6 +405,8 @@ export async function buildSeedableTestApp({ // own start still succeeds, so the node holds the app, but nothing that polls // docker afterwards can catch the process alive. if (exitAfterMs != null) environmentParameters.push(`EXIT_AFTER_MS=${exitAfterMs}`); + // a line on this interval, which docker writes to its data root as it comes + if (logEveryMs != null) environmentParameters.push(`LOG_EVERY_MS=${logEveryMs}`); const compose = [{ name, diff --git a/test-infra/runner/framework/silent-primary.js b/test-infra/runner/framework/silent-primary.js index ac9c0248a4..e50d453ce0 100644 --- a/test-infra/runner/framework/silent-primary.js +++ b/test-infra/runner/framework/silent-primary.js @@ -105,6 +105,7 @@ export async function placeSilentPrimaryApp({ return { env, + app, identifier, // When more than one node ran the component. twoRan, diff --git a/test-infra/runner/framework/state-events.js b/test-infra/runner/framework/state-events.js index e96462a747..60aab4dc6b 100644 --- a/test-infra/runner/framework/state-events.js +++ b/test-infra/runner/framework/state-events.js @@ -22,7 +22,9 @@ async function envelopeFor(nodeNum, data, broadcastedAt, signedBy) { }; } -// A v2 apprunning broadcast: the node runs `apps`, as of broadcastedAt. +// A v2 apprunning broadcast: the node runs `apps`, as of broadcastedAt. An app +// named by a string gets a hash and a runningSince; one given as an object is +// sent as it is, so an entry a node refuses can be built. export async function apprunningEvent({ nodeNum, apps, broadcastedAt, dedupKey = 'v2', signedBy = nodeNum, }) { @@ -30,11 +32,11 @@ export async function apprunningEvent({ const data = { type: 'fluxapprunning', version: 2, - apps: apps.map((name) => ({ - name, - hash: `hash-${name}`, + apps: apps.map((app) => (typeof app === 'string' ? { + name: app, + hash: `hash-${app}`, runningSince: new Date(broadcastedAt).toISOString(), - })), + } : app)), ip, broadcastedAt, osUptime: 10000, diff --git a/test-infra/runner/framework/syncthing-control.js b/test-infra/runner/framework/syncthing-control.js index 3325e69c2f..8a8be69ca3 100644 --- a/test-infra/runner/framework/syncthing-control.js +++ b/test-infra/runner/framework/syncthing-control.js @@ -356,3 +356,42 @@ export async function resetSyncState() { export async function setFolderPatchDelay({ ip = '*', ms = 0 }) { return post('/folder-patch-delay', { ip, ms }); } + +// A folder in a true cold-start shape on every holder: an empty global of its own +// and no connected peer holding the data. Set before install, so the first election +// sees it rather than the stub's default phantom-synced peer. Honest only for a bare +// g:/r: component: an f:/m:/ml: mount leaves FluxOS scaffolding on the volume that a +// real daemon counts and the stub cannot see, so those shapes run on syncthing: 'binary'. +export async function pinColdStart(holders, folder) { + const subnet = getSubnetConfig(); + await Promise.all(holders.map((i) => Promise.all([ + setSyncState({ + ip: subnet.nodeIp(i + 1), folder, state: 'idle', globalBytes: 0, inSyncBytes: 0, receiveOnlyChangedFiles: 0, + }), + setNoPeerData({ ip: subnet.nodeIp(i + 1), folder }), + ]))); +} + +// The owner's data on one holder, declared as ENTRIES rather than a count, because +// localHoldings totals the entry list and excludes the scaffolding in it by type and +// by name. Written after pinColdStart, which it overwrites for this one node. +export async function pinHolding(index, folder, bytes) { + const subnet = getSubnetConfig(); + const modified = new Date().toISOString(); + await setSyncState({ + ip: subnet.nodeIp(index + 1), + folder, + state: 'idle', + globalBytes: 0, + inSyncBytes: 0, + localChanged: [ + { + name: 'appdata', type: 'FILE_INFO_TYPE_DIRECTORY', size: 128, deleted: false, modified, + }, + { + name: 'appdata/seed-data', type: 'FILE_INFO_TYPE_FILE', size: bytes, deleted: false, modified, + }, + ], + }); + await setNoPeerData({ ip: subnet.nodeIp(index + 1), folder }); +} diff --git a/test-infra/runner/framework/syncthing-real.js b/test-infra/runner/framework/syncthing-real.js index 4831b2840b..ab2fae334a 100644 --- a/test-infra/runner/framework/syncthing-real.js +++ b/test-infra/runner/framework/syncthing-real.js @@ -1,7 +1,8 @@ // Reads a node's OWN syncthing daemon, for suites booted with // `createTestEnv({ syncthing: 'binary' })`. // -// Deliberately observers, not setters. The stub's control surface +// Observers, apart from the actions a machine itself takes on its daemon +// (start, stop, scan, pause a folder). The stub's control surface // (syncthing-control.js) scripts what a node believes about sync state, because // with a stub there is no truth to read. With a real daemon there is, and // scripting it would only hide what the test exists to establish - so these @@ -61,6 +62,13 @@ export async function getFolderStatus(client, folderId) { return api(client, `/rest/db/status?folder=${encodeURIComponent(folderId)}`); } +// How this node's syncthing sees another device's side of a folder: 'valid' while +// that device shares it over a live connection, 'paused', 'notSharing' or +// 'unknown' otherwise. What FluxOS reads a peer's connection to the folder from. +export async function getRemoteState(client, folderId, deviceId) { + return (await api(client, `/rest/db/completion?folder=${encodeURIComponent(folderId)}&device=${encodeURIComponent(deviceId)}`)).remoteState; +} + // A folder is complete when the index describes something and the disk holds all // of it. globalBytes === 0 means the index is empty, which is not the same as // synced and must never read as done. @@ -228,6 +236,16 @@ export async function stopDaemon(client) { await execInContainer(client.container, 'pkill -KILL -x syncthing --ns 1 --nslist pid; true'); } +// Pause or resume one folder, as FluxOS's pauseAllFolders does folder by folder - +// so a FluxOS that stops partway leaves some of a node's folders paused and the +// rest shared. Peers then read this node's connection to that folder as closed. +export async function setFolderPaused(client, folderId, paused) { + const key = await apiKey(client); + const r = await execInContainer(client.container, + `curl -sS -f -X PATCH -H "X-API-Key: ${key}" -H "Content-Type: application/json" -d '{"paused":${paused ? 'true' : 'false'}}' "http://127.0.0.1:8384/rest/config/folders/${encodeURIComponent(folderId)}"`); + if (r.exitCode !== 0) throw new Error(`syncthing-real: ${paused ? 'pause' : 'resume'} of ${folderId} failed: ${r.stderr || r.output}`); +} + // The daemon's own event log since `since`, optionally of named types - e.g. // StateChanged (a folder starting a scan) and ConfigSaved (a folder's type // changed). Event ids are ordered, so two events can be put in order, and diff --git a/test-infra/runner/framework/systemd-control.js b/test-infra/runner/framework/systemd-control.js index f466a16934..f95dcbaf25 100644 --- a/test-infra/runner/framework/systemd-control.js +++ b/test-infra/runner/framework/systemd-control.js @@ -1,10 +1,7 @@ -// Node control for systemd-mode nodes (createTestEnv({ systemdMode: true })). -// -// The default entrypoint's levers in container.js (restartFluxos, -// restartDockerd) drive a shell watchdog that does not exist here: under systemd -// FluxOS and dockerd are real units, so the equivalents are systemctl calls — -// which is also how a production host does it, making these the more faithful -// instruments of the two. +// Node control through systemd, which every node runs as PID 1: FluxOS and dockerd +// are units, controlled with systemctl as on a production host. container.js holds +// the faults (a FluxOS killed outright, a dockerd killed); these are the orderly +// stops and starts. // // The unit names mirror production: `fluxos` is the name an Arcane node's unit // carries. @@ -12,7 +9,7 @@ import { execInContainer } from './container.js'; import { waitFor } from './wait.js'; const FLUXOS_UNIT = 'fluxos'; -const DOCKERD_UNIT = 'dockerd'; +export const DOCKERD_UNIT = 'dockerd'; // Single-quote for `sh -c`: end the quote, escape the quote, reopen. const shQuote = (s) => `'${String(s).replace(/'/g, "'\\''")}'`; @@ -83,8 +80,9 @@ export async function startFluxos(container, { apiPort = 16127, readyTimeoutMs = } /** - * The systemd-mode restartFluxos: FluxOS's in-memory state is wiped while - * dockerd and the app containers keep running. Observably down, then up. + * `systemctl restart fluxos`, as an operator runs it: FluxOS stops in order and + * starts again, its in-memory state wiped while dockerd and the app containers keep + * running. Observably down, then up. */ export async function restartFluxos(container, opts = {}) { await stopFluxos(container, opts); @@ -105,6 +103,36 @@ export async function resumeDockerd(container, opts = {}) { { timeout: opts.timeout ?? 60000, interval: 500, label: 'dockerd reachable' }); } +/** The unit's cgroup directory, as the node's systemd reports it. Throws when it has none. */ +export async function unitCgroup(container, unit) { + const r = await execInContainer(container, `systemctl show -p ControlGroup --value ${unit}`); + const group = r.stdout.trim(); + if (r.exitCode !== 0 || !group) throw new Error(`unit ${unit} has no cgroup: is it running?`); + return `/sys/fs/cgroup${group}`; +} + +/** + * Stop every process of a unit where it stands, alive, until thawUnit: a service + * that cannot act at all, as a wedged or memory-starved one cannot. Writes the + * cgroup's freeze switch directly, because `systemctl freeze` waits for the freeze + * to complete, and a thread held in the kernel (on a frozen filesystem) completes + * it only once it is released. Such a thread stays held, unchanged, and every + * other thread of the unit stops at once. The env's teardown thaws it + * (host-stall.thawEveryNode). + */ +export async function freezeUnit(container, unit) { + const freeze = `${await unitCgroup(container, unit)}/cgroup.freeze`; + const r = await execInContainer(container, `echo 1 > ${freeze} && cat ${freeze}`); + if (r.exitCode !== 0 || r.stdout.trim() !== '1') throw new Error(`freezeUnit ${unit}: ${(r.stderr || r.stdout).trim()}`); +} + +/** Let a unit frozen by freezeUnit run again. */ +export async function thawUnit(container, unit) { + const freeze = `${await unitCgroup(container, unit)}/cgroup.freeze`; + const r = await execInContainer(container, `echo 0 > ${freeze} && cat ${freeze}`); + if (r.exitCode !== 0 || r.stdout.trim() !== '0') throw new Error(`thawUnit ${unit}: ${(r.stderr || r.stdout).trim()}`); +} + /** * Read a unit's journal. `processOnly` restricts to the unit's own output * stream (`_TRANSPORT=stdout`, which carries both stdout and stderr) — without diff --git a/test-infra/runner/framework/test-env.js b/test-infra/runner/framework/test-env.js index 2a269a7a09..f944b7e142 100644 --- a/test-infra/runner/framework/test-env.js +++ b/test-infra/runner/framework/test-env.js @@ -6,7 +6,7 @@ process.env.TESTCONTAINERS_RYUK_RECONNECTION_TIMEOUT ??= '5s'; import { GenericContainer, Wait, getContainerRuntimeClient } from 'testcontainers'; import { - readFileSync, mkdirSync, writeFileSync, rmSync, existsSync, + readFileSync, mkdirSync, mkdtempSync, writeFileSync, rmSync, existsSync, } from 'node:fs'; import { execFile, execFileSync } from 'node:child_process'; import { tmpdir } from 'node:os'; @@ -14,7 +14,8 @@ import { join, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import http from 'node:http'; import { nodeClient } from './node-client.js'; -import { execInContainer } from './container.js'; +import { execInContainer, restartFluxos } from './container.js'; +import { thawEveryNode } from './host-stall.js'; import { waitFor } from './wait.js'; import { HttpPollWaitStrategy } from './http-wait-strategy.js'; import { TcpPollWaitStrategy } from './tcp-wait-strategy.js'; @@ -42,6 +43,14 @@ import { startSyncedDataKeeper } from './synced-data-keeper.js'; // replaying a line it already has. Docker's `since` is whole-second, so the replay is over // within a second of the stream opening; this is that bound with room for a loaded box. const REPLAY_WINDOW_MS = 5000; +// Bounded docker CLI call from the runner host. +function dockerCli(args, timeoutMs = 15000, maxBuffer = 8 * 1024 * 1024) { + return new Promise((resolve) => { + execFile('docker', args, { timeout: timeoutMs, maxBuffer }, (err, stdout, stderr) => { + resolve({ ok: !err, stdout: stdout ?? '', stderr: stderr ?? '' }); + }); + }); +} // A node's whole record, never a tail: the journal directory is the node's own, so // it holds exactly this node's entries since its container was created, and a @@ -208,6 +217,71 @@ const IMAGE_TAG = process.env.FLUX_E2E_TAG || 'latest'; const image = (name) => `${name}:${IMAGE_TAG}`; +const NODE_IMAGE = 'flux-e2e-fluxos-01'; + +// The node image a node boots: the run's tag, or the tag a suite names for that +// node, so a fleet can mix commits. An image under another tag is built from +// its own tree, which is not here to check it against, so what can be required +// is that it exists and was stamped by build-images.sh. +const nodeImageRef = (tag) => `${NODE_IMAGE}:${tag}`; + +async function assertNodeImageStamped(tag) { + const ref = nodeImageRef(tag); + const res = await dockerCli(['image', 'inspect', ref, '--format', '{{index .Config.Labels "flux.e2e.src"}}']); + if (!res.ok) { + throw new Error(`node image ${ref} is not built on this host: build it from its commit's clone with FLUX_E2E_TAG=${tag} ./test-infra/build-images.sh fluxos-01`); + } + const label = res.stdout.trim(); + if (!label || label === '') throw new Error(`node image ${ref} carries no source stamp: build it with build-images.sh, never by hand`); + return label; +} + +// The FluxOS code of one built node image, copied into a running node: the +// parts of the tree the process loads. Its data, its containers and its +// database stay. +const UPGRADE_DIRS = ['ZelBack/src', 'helpers', 'lib']; +const UPGRADE_FILES = ['package.json', 'ZelBack/pinEnvironment.js', 'ZelBack/config/default.js', 'ZelBack/config/volumeToolsImage.json', 'app.js', 'apiServer.js', 'init.js']; +// The new build's per-node config files and the script that writes the overlay from them. +const UPGRADE_HARNESS = ['test-infra/config', 'test-infra/write-node-config.sh']; + +// A digest of the FluxOS code a running node holds: the path and content of every +// file the process loads, so two builds of the same code agree whatever their +// version, tag or file times. +async function fluxosCodeDigest(container) { + const paths = [...UPGRADE_DIRS, ...UPGRADE_FILES].join(' '); + const res = await execInContainer(container, `cd /flux && find ${paths} -type f -print0 | sort -z | xargs -0 sha256sum | sha256sum`); + const digest = res.stdout.trim().split(/\s+/)[0]; + if (res.exitCode !== 0 || !/^[0-9a-f]{64}$/.test(digest)) throw new Error(`cannot read the FluxOS code of ${container.getId()}: ${res.output}`); + return digest; +} + +// A fleet that names more than one image runs more than one FluxOS: every tag's +// nodes hold code no other tag's nodes hold. +async function assertTagsRunOtherCode(fluxNodes) { + const byTag = new Map(); + for (const node of fluxNodes) { + if (node.container && !byTag.has(node.imageTag)) byTag.set(node.imageTag, node.container); + } + if (byTag.size < 2) return; + const tags = [...byTag.keys()]; + const digests = await Promise.all(tags.map((tag) => fluxosCodeDigest(byTag.get(tag)))); + const tagOf = new Map(); + digests.forEach((digest, i) => { + if (tagOf.has(digest)) throw new Error(`node images ${nodeImageRef(tagOf.get(digest))} and ${nodeImageRef(tags[i])} hold the same FluxOS code, so the fleet is not mixed`); + tagOf.set(digest, tags[i]); + }); +} + +async function dockerCp(from, to) { + const res = await dockerCli(['cp', from, to], 180000); + if (!res.ok) throw new Error(`docker cp ${from} ${to}: ${res.stderr.trim()}`); +} + +function dependenciesOf(packageJsonPath) { + const pkg = JSON.parse(readFileSync(packageJsonPath, 'utf8')); + return JSON.stringify({ dependencies: pkg.dependencies ?? {}, optionalDependencies: pkg.optionalDependencies ?? {} }); +} + // Where this run's log output lives. run-all.sh exports the resolved value; a // standalone `npx mocha` has no runner to inherit from and gets the same default // the runner would have applied, so both paths write to one place. @@ -322,10 +396,9 @@ class StaticIpContainer extends GenericContainer { return this; } - // Per-container stop signal (the image's STOPSIGNAL stays SIGTERM for the - // default entrypoint). systemd-mode nodes need SIGRTMIN+3: systemd as PID 1 - // treats SIGTERM as a reexec request, so a plain docker stop would sit out - // the full kill timeout on every teardown. + // Per-container stop signal. A node needs SIGRTMIN+3: systemd as PID 1 treats + // SIGTERM as a reexec request, so a plain docker stop would sit out the full kill + // timeout on every teardown. withStopSignal(signal) { this.#stopSignal = signal; return this; @@ -588,9 +661,8 @@ function makeEnvShell(networkName) { return [...byIndex.values()].sort((a, b) => a.index - b.index); }, - // Each systemd-mode node's journal, read from the host copy, so a node whose - // container is already gone - one that never finished booting - still has its - // record. Empty for a default-mode env, whose nodes log to the container stream. + // Each node's journal, read from the host copy, so a node whose container is + // already gone - one that never finished booting - still has its record. async nodeJournals() { return Promise.all(nodeConfigs .filter((cfg) => cfg.journalDir && existsSync(cfg.journalDir)) @@ -664,6 +736,8 @@ function makeEnvShell(networkName) { // the volume (EPERM) and every run leaks its node volumes. Unmount to // expose the bare dirs and strip the flag while the node is still // running; containers going down makes this the last chance to exec. + await thawEveryNode(clients); + step('thaw'); await Promise.all(clients.map(async (client) => { if (!client?.container) return; await execInContainer( @@ -925,11 +999,7 @@ export async function createTestEnv({ tickerAutostart = false, discoveryAutostart = false, nodeStatusOverrides = {}, rpcFailures = [], bootContext = 'running', initialHeight = DEFAULT_INITIAL_HEIGHT, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation = {}, locationTable = null, staticIp = true, policy = null, policySeeds = null, - awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], - // systemd-mode nodes: the entrypoint execs a real systemd as PID 1, so dockerd - // and fluxos run as units and FluxOS's output goes to the journal. Env-level - // and uniform across nodes. - systemdMode = false, + awaitPolicy = true, pm2Nodes = {}, pausedSyncthingNodes = [], syncthingFaults = {}, sharedKeys = {}, collateralOf = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = [], rejoinOnRestart = true, networkShapes = {}, globalIpv6 = [], dnsRecords = [], // Firewalled nodes boot with ufw active as their install leaves it - the legacy // installer's rules on a legacy node, the ISO's on an Arcane one - and FluxOS // adds its own on top. true for every node, or a list of node indices. Off @@ -941,6 +1011,11 @@ export async function createTestEnv({ } // Arcane nodes whose OS starts syncthing paused, as a syncthing.service with // --paused does. A legacy node's syncthing is FluxOS's to start, always paused. + // Nodes whose docker keeps its data on a filesystem a suite can freeze + // (framework/host-stall.js), by node index. + for (const index of freezableDockerRoot) { + if (!Number.isInteger(index) || index < 0 || index >= nodes) throw new Error(`createTestEnv: freezableDockerRoot names node ${index}, outside a fleet of ${nodes}`); + } for (const index of pausedSyncthingNodes) { if (syncthing !== 'binary') throw new Error('createTestEnv: pausedSyncthingNodes needs syncthing: \'binary\''); if (legacyNodes.includes(index)) throw new Error(`createTestEnv: paused syncthing node ${index} is legacy, whose syncthing FluxOS starts`); @@ -1192,12 +1267,6 @@ export async function createTestEnv({ if (aptBadSource && legacyNodes.length === 0) { throw new Error('createTestEnv: aptBadSource needs legacyNodes - no other node type runs apt'); } - // fluxos.service runs FluxOS as root, so an unprivileged node cannot be one. - // Refused here rather than at the node's own boot, where it would read as a - // node that never came up. - if (systemdMode && unprivilegedNodes.length) { - throw new Error('createTestEnv: systemdMode runs FluxOS as root - unprivilegedNodes is not supported with it'); - } // The boot-lock queue wait must not count against the suite's hook budget. // Mocha enforces a hook's timeout twice: the watchdog timer (which would fire // MID-QUEUE whenever the queue alone outlasts the budget), and a completion-time @@ -1237,7 +1306,7 @@ export async function createTestEnv({ // mongo starts, i.e. inside the fleet boot, where the waits at risk are the // boot's own. await startInfraDeathWatch(env); - await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, systemdMode, firewall, collateralOf, syncthingFaults, configuredIp); + await _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, mergedNodeOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing, aptSeeded, aptBadSource, geolocation, locationTable, staticIp, policy, policySeeds, policyReachable, pm2Nodes, pausedSyncthingNodes, { ...sharedKeys, ...collateralOf }, networkShapes, globalIpv6, dnsRecords, firewall, collateralOf, syncthingFaults, configuredIp, nodeImageTags, freezableDockerRoot); return env; } catch (err) { // Boot failed: the env owns everything started so far. The shared teardown @@ -1266,7 +1335,7 @@ function mergeConfigs(base, override) { return result; } -async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], systemdMode = false, firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}) { +async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNodes, stubPeers, silentSyncPeers, unverifiableSyncPeers, policyUnawarePeers, stubPeerings, configOverrides, nodeConfigOverrides, nodeTiers, dataCenter, tickerAutostart, discoveryAutostart, nodeStatusOverrides, rpcFailures, bootContext, initialHeight, syncthing = 'stub', aptSeeded = true, aptBadSource = false, geolocation, locationTable, staticIp = true, policy = null, policySeeds = null, policyReachable = false, pm2Nodes = {}, pausedSyncthingNodes = [], sharedKeys = {}, networkShapes = {}, globalIpv6 = [], dnsRecords = [], firewall = false, collateralOf = {}, syncthingFaults = {}, configuredIp = {}, nodeImageTags = {}, freezableDockerRoot = []) { // Everything built here registers onto the env shell as it comes up, so a // boot-phase throw leaves the partial state reachable (see makeEnvShell). const { @@ -1478,11 +1547,14 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // Takes the stub's own /policy control body: { documents }, { signer: 'rogue' }, // { seq }, { available: false } to boot a fleet whose source never answers. if (policy) { - await fetch(`http://${EXTERNAL_STUB_IP}:3001/policy`, { + const published = await fetch(`http://${EXTERNAL_STUB_IP}:3001/policy`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(policy), }); + if (!published.ok) { + throw new Error(`createTestEnv: the stub refused the policy (${published.status}): ${await published.text()}`); + } } // Names the fleet's resolver answers itself, set before any node starts so the first @@ -1548,6 +1620,8 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod const deferredBuilders = new Map(); const firstDeferred = nodes - deferredNodes; + // A node booting another commit's image needs that image here, stamped. + await Promise.all([...new Set(Object.values(nodeImageTags))].filter((t) => t !== IMAGE_TAG).map(assertNodeImageStamped)); for (let i = 0; i < nodes; i++) { if (stubPeerSet.has(i)) continue; @@ -1575,7 +1649,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // the rest of the run's record. // Kept for passing suites too: a green suite is exactly where a node that died // and was never noticed would hide, and this record cannot be reconstructed later. - const journalDir = systemdMode ? join(runLogDir(), `journal-${networkName}-${num}`) : null; + const journalDir = join(runLogDir(), `journal-${networkName}-${num}`); if (journalDir) { mkdirSync(journalDir, { recursive: true }); bindMounts.push( @@ -1627,6 +1701,7 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod } if (!isLegacy) nodeEnv.FLUXOS_PATH = '/flux'; if (pausedSyncthingNodes.includes(i)) nodeEnv.FLUX_SYNCTHING_PAUSED = '1'; + if (freezableDockerRoot.includes(i)) nodeEnv.FLUX_FREEZABLE_DOCKER_ROOT = 'true'; if (syncthingFaults[i]) nodeEnv.FLUX_SYNCTHING_FAULT = syncthingFaults[i]; // WHETHER THIS NODE'S FLUXOS IS ROOT. Declared per node rather than derived from // legacy, because the field carries both: Arcane runs FluxOS as root, and an @@ -1635,8 +1710,10 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // can READ - syncthing is spawned with sudo whichever account FluxOS holds, so on // an unprivileged node the index can describe files the node itself is refused. if (unprivilegedNodes.includes(i)) nodeEnv.FLUX_FLUXOS_USER = 'fluxuser'; + // pm2 runs as the account the multitool installed under, and FluxOS as pm2 does. if (Object.prototype.hasOwnProperty.call(pm2Nodes, i)) { nodeEnv.FLUX_PM2 = '1'; + nodeEnv.FLUX_FLUXOS_USER = 'fluxuser'; if (pm2Nodes[i] != null) nodeEnv.FLUX_PM2_KILL_TIMEOUT_MS = String(pm2Nodes[i]); } // Legacy only, because it is the only node type that installs anything: @@ -1644,7 +1721,6 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // of syncthing would simply never get it back. if (!aptSeeded && isLegacy) nodeEnv.FLUX_APT_SEEDED = 'false'; if (aptBadSource && isLegacy) nodeEnv.FLUX_APT_BAD_SOURCE = 'true'; - if (systemdMode) nodeEnv.FLUX_SYSTEMD_MODE = 'true'; if (firewall === true || (Array.isArray(firewall) && firewall.includes(i))) nodeEnv.FLUX_FIREWALL = 'true'; // Point the node's config at the base-derived infra IPs. The mounted config // files carry the default 31.200.0 addresses; this is written into the node's @@ -1751,7 +1827,8 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // state machine: under a contended 10-node fleet boot, Wait.forHealthCheck() // tears the fleet down on a transient "unhealthy" even when FluxOS is up. See // http-wait-strategy.js for the full rationale. - const builder = new StaticIpContainer(image('flux-e2e-fluxos-01')) + const imageTag = nodeImageTags[i] ?? IMAGE_TAG; + const builder = new StaticIpContainer(nodeImageRef(imageTag)) .withPrivilegedMode() .withStaticIp(networkName, nodeIp) // Nodes resolve through the stub, which answers fleet names by relaying to @@ -1765,11 +1842,11 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod .withLogConsumer(logCollector) .withEnvironment(nodeEnv) .withWaitStrategy(nodeReadyWaitStrategy(nodeIp).withStartupTimeout(120000)); - if (systemdMode) builder.withStopSignal('SIGRTMIN+3'); + builder.withStopSignal('SIGRTMIN+3'); if (nodeAppArmor()) builder.withAppArmorProfile(nodeAppArmor()); nodeConfigs.push({ - index: i, builder, ip: nodeIp, num: i + 1, logCollector, bootIdDir, journalDir, + index: i, builder, ip: nodeIp, num: i + 1, logCollector, bootIdDir, journalDir, imageTag, }); } @@ -1818,18 +1895,25 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod for (let i = 0; i < nodes; i++) { const cfg = fluxNodesByIndex.get(i); if (!cfg) { - fluxNodes.push({ container: null, ip: subnet.nodeIp(i + 1), num: i + 1, logCollector: null, bootIdDir: null }); + fluxNodes.push({ + container: null, ip: subnet.nodeIp(i + 1), num: i + 1, logCollector: null, bootIdDir: null, imageTag: IMAGE_TAG, + }); continue; } const s = startedByIndex.get(i); if (s) { - fluxNodes.push({ container: s.container, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir }); + fluxNodes.push({ + container: s.container, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir, imageTag: cfg.imageTag, + }); } else { deferredBuilders.set(i, cfg.builder); - fluxNodes.push({ container: null, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir }); + fluxNodes.push({ + container: null, ip: cfg.ip, num: cfg.num, logCollector: cfg.logCollector, bootIdDir: cfg.bootIdDir, imageTag: cfg.imageTag, + }); } } containers.fluxNodes = fluxNodes; + await assertTagsRunOtherCode(fluxNodes); for (const n of fluxNodes) { if (!n.container) { @@ -1911,18 +1995,6 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod // Nodes this fixture has cut off from the fleet. They reach nobody and nobody reaches // them, so they peer with nobody and obtain no policy - by the fixture's own doing. const heldOut = new Set(); - // In systemd mode the container's stdout is systemd's console and FluxOS's - // own stream goes to the journal, so the log collectors legitimately receive - // nothing from FluxOS. Answering "no match" would leave every assertion built - // on these silently unfalsifiable, so refuse and name the replacement. - const assertCollectorSeesFluxos = (fn) => { - if (!systemdMode) return; - throw new Error( - `${fn}() cannot see FluxOS logs in systemd mode: its stdout goes to the journal, ` - + 'not the container stream. Read the journal instead — journalGrep(container, ' - + "'fluxos', pattern, { processOnly: true }) from framework/systemd-control.js", - ); - }; // Post-boot methods join the shell here (they close over _buildEnv locals like // deferredBuilders/fluxNodes); identity, registries and teardown live on the @@ -1967,6 +2039,67 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod return client; }, + // The image tag a node's FluxOS code came from: the one it booted, or the + // one it was last upgraded to. + nodeImageTag(index) { + return fluxNodes[index]?.imageTag ?? IMAGE_TAG; + }, + + // Make a running node run the FluxOS of another built image and restart + // FluxOS on it, as a node's own update does: its data, containers and database + // stay; its code, its installed packages when the build declares others, and + // its config overlay become the new build's - the overlay written from that + // build's per-node file and this run's overrides, by the script the entrypoint + // runs at boot. Every migration runs at the first start of the new code. + // Upgrade or downgrade alike. + async upgradeNode(index, { tag, readyTimeoutMs = 180000 } = {}) { + const node = fluxNodes[index]; + if (!node?.container) throw new Error(`upgradeNode: node ${index} is not running`); + if (!tag) throw new Error('upgradeNode: the image tag to run is required'); + await assertNodeImageStamped(tag); + const nodeId = node.container.getId(); + const codeBefore = await fluxosCodeDigest(node.container); + const created = await dockerCli(['create', nodeImageRef(tag)], 60000); + if (!created.ok) throw new Error(`upgradeNode: cannot read ${nodeImageRef(tag)}: ${created.stderr.trim()}`); + const sourceId = created.stdout.trim(); + const host = mkdtempSync(join(tmpdir(), 'flux-e2e-upgrade-')); + try { + await dockerCp(`${sourceId}:/flux/package.json`, `${host}/source-package.json`); + await dockerCp(`${nodeId}:/flux/package.json`, `${host}/node-package.json`); + const packagesChange = dependenciesOf(`${host}/source-package.json`) !== dependenciesOf(`${host}/node-package.json`); + const replaced = [...UPGRADE_DIRS, ...(packagesChange ? ['node_modules'] : [])]; + mkdirSync(`${host}/flux/ZelBack/config`, { recursive: true }); + mkdirSync(`${host}/flux/test-infra`, { recursive: true }); + for (const part of [...replaced, ...UPGRADE_FILES, ...UPGRADE_HARNESS]) { + // eslint-disable-next-line no-await-in-loop + await dockerCp(`${sourceId}:/flux/${part}`, `${host}/flux/${part}`); + } + const clear = [...replaced, ...UPGRADE_HARNESS].map((d) => `rm -rf /flux/${d}`).join(' && '); + const cleared = await execInContainer(node.container, clear); + if (cleared.exitCode !== 0) throw new Error(`upgradeNode: cannot clear node ${index}'s code: ${cleared.output}`); + for (const part of [...replaced, ...UPGRADE_HARNESS]) { + // eslint-disable-next-line no-await-in-loop + await dockerCp(`${host}/flux/${part}`, `${nodeId}:/flux/${dirname(part)}/`); + } + for (const file of UPGRADE_FILES) { + // eslint-disable-next-line no-await-in-loop + await dockerCp(`${host}/flux/${file}`, `${nodeId}:/flux/${file}`); + } + const owned = [...replaced, ...UPGRADE_FILES].map((p) => `/flux/${p}`).join(' '); + const configured = await execInContainer(node.container, `/flux/test-infra/write-node-config.sh && chown -R fluxuser:fluxuser ${owned} /flux/ZelBack/config && (chown fluxuser:fluxuser /flux/ZelBack/shared.js 2>/dev/null || true)`); + if (configured.exitCode !== 0) throw new Error(`upgradeNode: cannot write node ${index}'s config: ${configured.output}`); + } finally { + await dockerCli(['rm', '-f', sourceId], 60000); + rmSync(host, { recursive: true, force: true }); + } + if (await fluxosCodeDigest(node.container) === codeBefore) { + throw new Error(`upgradeNode: ${nodeImageRef(tag)} holds the FluxOS code node ${index} already ran, so nothing was upgraded`); + } + await restartFluxos(node.container, { readyTimeoutMs }); + node.imageTag = tag; + }, + + // Wait on an HTTP poll of /flux/version rather than Docker's health state // machine: on restart Docker transiently reports "unhealthy" during monitor // teardown (moby/daemon/container/health.go CloseMonitorChannel), which a @@ -2255,17 +2388,14 @@ async function _buildEnv(env, nodes, deferredNodes, legacyNodes, unprivilegedNod }, nodeHasLog(index, pattern) { - assertCollectorSeesFluxos('nodeHasLog'); return fluxNodes[index].logCollector.hasLine(pattern); }, nodeLogCount(index, pattern) { - assertCollectorSeesFluxos('nodeLogCount'); return fluxNodes[index].logCollector.countPattern(pattern); }, nodeLogLines(index) { - assertCollectorSeesFluxos('nodeLogLines'); return fluxNodes[index].logCollector.getLines(); }, }); diff --git a/test-infra/runner/framework/volume-fixture.js b/test-infra/runner/framework/volume-fixture.js index c4a0f8a028..b76fcc3eed 100644 --- a/test-infra/runner/framework/volume-fixture.js +++ b/test-infra/runner/framework/volume-fixture.js @@ -26,6 +26,8 @@ export const APP_GID = 33; export const appId = (name) => `flux${name}_${name}`; export const volumeRoot = (name) => `/mnt/appdata/flux-apps/${appId(name)}`; export const volumeImage = (name) => `/mnt/appdata/${appId(name)}FLUXFSVOL`; +const folderRoot = (folderId) => `/mnt/appdata/flux-apps/${folderId}`; +const folderImage = (folderId) => `/mnt/appdata/${folderId}FLUXFSVOL`; // Single-quote for the shell. Fixture content is chosen by the suites, but a // literal $ or backtick in one must stay a literal rather than becoming @@ -43,12 +45,59 @@ async function run(container, command, what) { } /** - * Takes an app's volume away for good: unmounted, its directory no longer + * Takes a component's volume away for good: unmounted, its directory no longer * immutable, and its image removed, so nothing on the node can mount it again. + * @param {string} folderId The component's identifier, flux_. + */ +export async function destroyFolderVolume(container, folderId) { + const root = folderRoot(folderId); + await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)} && rm -f ${sq(folderImage(folderId))}`, `destroy the volume of ${folderId}`); +} + +const folderImageHeader = (folderId) => `${folderImage(folderId)}.header`; + +/** + * Leaves a component's volume image on the node with nothing able to mount it: + * unmounted, its directory no longer immutable, and the image's first 4 KiB, which + * carry its filesystem's superblock, zeroed. The bytes are kept beside the image for + * restoreFolderVolume. Waits for the loop device to release the image first, so no + * mount still open on it writes the superblock back. + * @param {string} folderId The component's identifier, flux_. + */ +export async function breakFolderVolume(container, folderId) { + const root = folderRoot(folderId); + const image = folderImage(folderId); + await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)}`, `unmount the volume of ${folderId}`); + await run( + container, + `for i in $(seq 1 60); do [ -z "$(losetup -j ${sq(image)})" ] && exit 0; sleep 1; done; losetup -j ${sq(image)}; exit 1`, + `wait for the loop device to release the image of ${folderId}`, + ); + await run( + container, + `dd if=${sq(image)} of=${sq(folderImageHeader(folderId))} bs=4096 count=1 status=none && dd if=/dev/zero of=${sq(image)} bs=4096 count=1 conv=notrunc status=none`, + `zero the superblock of ${folderId}`, + ); + const probe = await execInContainer(container, `blkid -p ${sq(image)}`); + if (probe.exitCode === 0) throw new Error(`breakFolderVolume: ${image} still reads as a filesystem: ${probe.output}`); +} + +/** + * Puts back what breakFolderVolume zeroed, so the image mounts again. + * @param {string} folderId The component's identifier, flux_. + */ +export async function restoreFolderVolume(container, folderId) { + const image = folderImage(folderId); + const header = folderImageHeader(folderId); + await run(container, `dd if=${sq(header)} of=${sq(image)} bs=4096 count=1 conv=notrunc status=none && rm -f ${sq(header)}`, `restore the superblock of ${folderId}`); + await run(container, `blkid -p ${sq(image)}`, `read the restored image of ${folderId} as a filesystem`); +} + +/** + * destroyFolderVolume for an app whose one component shares its name. */ export async function destroyVolume(container, name) { - const root = volumeRoot(name); - await run(container, `umount -l ${sq(root)} && chattr -i ${sq(root)} && rm -f ${sq(volumeImage(name))}`, `destroy the volume of ${name}`); + await destroyFolderVolume(container, appId(name)); } /** diff --git a/test-infra/runner/framework/wait.js b/test-infra/runner/framework/wait.js index 9d6ada4810..73a77cf552 100644 --- a/test-infra/runner/framework/wait.js +++ b/test-infra/runner/framework/wait.js @@ -12,7 +12,7 @@ const { InstallOutcome } = installOutcomeModule; // timeout that names the wrong cause. const HOLDS_THE_APP = new Set([InstallOutcome.INSTALLED, InstallOutcome.ALREADY_INSTALLED]); const LACKS_THE_APP = new Set([InstallOutcome.DECLINED, InstallOutcome.FAILED]); -const DECIDES_NOTHING = new Set([InstallOutcome.BUSY]); +const DECIDES_NOTHING = new Set([InstallOutcome.BUSY, InstallOutcome.DEFERRED]); const unclassified = Object.values(InstallOutcome).filter( (outcome) => !HOLDS_THE_APP.has(outcome) && !LACKS_THE_APP.has(outcome) && !DECIDES_NOTHING.has(outcome), diff --git a/test-infra/runner/run-parallel.sh b/test-infra/runner/run-parallel.sh index 0e2c7f7fde..766587b6a5 100755 --- a/test-infra/runner/run-parallel.sh +++ b/test-infra/runner/run-parallel.sh @@ -129,14 +129,9 @@ fi # - the silent exit-255 class that went unexplained for three gates and survived # six reproduction attempts, because a solo run never exhausts a host-wide pool. # -# The arithmetic is why it was intermittent rather than constant. Measured directly -# once the gauge existed (gate 10): a full gate peaks at 102 concurrent containers -# and 221 inotify instances - 2.17 each, not the 1.27 a quieter sample suggested. -# So the harness has been running at 1.7x the default ceiling of 128, and the wonder -# is not that gates failed but that any passed: containers hold an instance only -# briefly, so a boot survived or died on the exact overlap. 1024 is 4.6x the measured -# peak, which leaves room for wider fleets rather than merely clearing today's number. -INOTIFY_MIN="${E2E_INOTIFY_MAX_USER_INSTANCES:-1024}" +# Every node's systemd holds instances for as long as the node runs, so the pool is +# sized far above any gate's need: an instance costs only a little kernel memory. +INOTIFY_MIN="${E2E_INOTIFY_MAX_USER_INSTANCES:-8192}" require_int E2E_INOTIFY_MAX_USER_INSTANCES "$INOTIFY_MIN" ino_now=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0) require_int fs.inotify.max_user_instances "$ino_now" @@ -145,7 +140,7 @@ if [ "$ino_now" -lt "$INOTIFY_MIN" ]; then ino_now=$(sysctl -n fs.inotify.max_user_instances 2>/dev/null || echo 0) if [ "$ino_now" -lt "$INOTIFY_MIN" ]; then abort 94 "###ABORT inotify instance pool too small (max_user_instances=$ino_now < $INOTIFY_MIN) and could not raise it." \ - "Every systemd-mode node needs one and the pool is host-wide, so a gate on this host" \ + "Every node's systemd needs one and the pool is host-wide, so a gate on this host" \ "produces silent exit-255 boot deaths by design." \ "Raise it for the run: sudo sysctl -w fs.inotify.max_user_instances=$INOTIFY_MIN" \ "Or lower the bar deliberately with E2E_INOTIFY_MAX_USER_INSTANCES." @@ -175,6 +170,22 @@ if [ -z "${SUITES:-}" ]; then SUITES="$heavy $light" fi +# A suite that boots a node on another build of FluxOS declares it on a line of its +# own (`// base-image: FLUX_E2E_BASE_TAG`); the gate refuses before launching anything +# when a chosen suite declares one and that image is not on this host, rather than +# letting the suite fail inside the run. +for s in $SUITES; do + grep -qh '^// base-image: FLUX_E2E_BASE_TAG$' tests/${s}-*.js 2>/dev/null || continue + if [ -z "${FLUX_E2E_BASE_TAG:-}" ]; then + abort 91 "###ABORT suite $s boots a node on a base image and FLUX_E2E_BASE_TAG is not set." \ + "Build one: FLUX_E2E_TAG= ./test-infra/build-release-image.sh , then run with FLUX_E2E_BASE_TAG=." + fi + if ! docker image inspect "flux-e2e-fluxos-01:${FLUX_E2E_BASE_TAG}" >/dev/null 2>&1; then + abort 91 "###ABORT suite $s boots a node on flux-e2e-fluxos-01:${FLUX_E2E_BASE_TAG}, which is not on this host." \ + "Build it: FLUX_E2E_TAG=${FLUX_E2E_BASE_TAG} ./test-infra/build-release-image.sh ." + fi +done + free_mb(){ free -m | awk '/^Mem:/{print $7}'; } load_1m(){ cut -d' ' -f1 /proc/loadavg | cut -d. -f1; } # integer part is enough for a gate diff --git a/test-infra/runner/tests/02-boot-prerequisites.js b/test-infra/runner/tests/02-boot-prerequisites.js index 5c67e993d6..182196b554 100644 --- a/test-infra/runner/tests/02-boot-prerequisites.js +++ b/test-infra/runner/tests/02-boot-prerequisites.js @@ -3,6 +3,7 @@ import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { waitForDaemonReady, waitFor } from '../framework/wait.js'; import { dbClient } from '../framework/db-client.js'; +import { execInContainer } from '../framework/container.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; @@ -67,7 +68,9 @@ describe('Boot: prerequisites', function () { }); it('should start Docker daemon inside container', async function () { - expect(env.nodeHasLog(0, 'dockerd is ready')).to.equal(true); + const unit = await execInContainer(env.clients[0].container, 'systemctl is-active dockerd'); + expect(unit.stdout.trim()).to.equal('active'); + expect(env.nodeHasLog(0, 'Docker daemon connected')).to.equal(true); }); it('should initialize app spawner', async function () { diff --git a/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js b/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js index 74e34b6e95..8e6f18b172 100644 --- a/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js +++ b/test-infra/runner/tests/1502-policy-reaches-a-node-from-its-peers.js @@ -10,7 +10,7 @@ import { getSubnetConfig } from '../framework/subnet-config.js'; import { waitFor, waitForBootSettled } from '../framework/wait.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; import { buildAppSpec } from '../framework/app-helper.js'; -import { execInContainer } from '../framework/container.js'; +import { execInContainer, fluxosPid } from '../framework/container.js'; // Policy arriving from a PEER rather than from the published source. // @@ -175,12 +175,12 @@ describe('policy reaching a node from its peers', function () { // message, so a failure inside it has nowhere to go but the process - and the node's // answer to that is to exit. Sent from the stub exactly as a real peer would. const client = env.clients[TOLD_NODE]; - const readPid = async () => (await execInContainer(client.container, 'cat /tmp/fluxos.pid')).stdout.trim(); + const readPid = () => fluxosPid(client.container); const pidBefore = await readPid(); // A pid that was never read is equal to itself afterwards, so the comparison below // would hold for a node that had died and come back twice. - expect(pidBefore, 'no FluxOS pid to compare against').to.match(/^[0-9]+$/); + expect(pidBefore, 'no FluxOS pid to compare against').to.be.a('number'); const seqBefore = await heldSeq(TOLD_NODE); // Not text. Measuring it raises before anything is parsed and before any signature is diff --git a/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js b/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js index 1f4d8e9497..01d45ae7d4 100644 --- a/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js +++ b/test-infra/runner/tests/1802-mount-safety-on-an-unprivileged-node.js @@ -1,7 +1,7 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { execInContainer, getAppContainerStatus } from '../framework/container.js'; +import { execInContainer, getAppContainerStatus, fluxosPid } from '../framework/container.js'; import { setSyncState, setSynced, setSyncing, getSyncthingState, resetSyncState, injectSyncthingEvent, @@ -48,10 +48,10 @@ async function folderType(nodeIp, folderId) { return node?.folders?.find((f) => f.id === folderId)?.type ?? null; } -// The account the node's own FluxOS process holds, read off the process the -// entrypoint supervises rather than off the container, which stays root. +// The account the node's own FluxOS process holds, read off the process rather +// than off the container, which stays root. async function fluxosAccount(client) { - const r = await execInContainer(client.container, 'sh -c \'ps -o user= -p "$(cat /tmp/fluxos.pid)"\''); + const r = await execInContainer(client.container, `ps -o user= -p ${await fluxosPid(client.container)}`); return r.output.trim(); } diff --git a/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js b/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js index bae3749b50..04d978d7dc 100644 --- a/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js +++ b/test-infra/runner/tests/2202-single-writer-election-on-real-syncthing.js @@ -38,6 +38,12 @@ const APP_UID = 1000; const API_PORT = 16127; const SYNCTHING_PORT = API_PORT + 2; +// The old primary is down through the whole takeover and comes back as a standby, +// which a node does only while its locations have not expired: past that its boot +// removes every app. The shared harness fleet's 63s is shorter than a takeover can +// take; production's 125 min is not, and this fleet keeps production's order. +const LOCATION_TTL_S = 30 * 60; + describe('single-writer election on real syncthing', function () { let env; dumpLogsOnFailure(() => env); @@ -78,7 +84,7 @@ describe('single-writer election on real syncthing', function () { syncthing: 'binary', tickerAutostart: false, configOverrides: { - fluxapps: { minOutgoing: 1, minIncoming: 1 }, + fluxapps: { minOutgoing: 1, minIncoming: 1, locationTtlS: LOCATION_TTL_S }, }, }); await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); diff --git a/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js b/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js index 0cc93653a6..b3f5680fbb 100644 --- a/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js +++ b/test-infra/runner/tests/2208-legacy-pm2-kill-timeout-request-never-strands-the-boot.js @@ -1,7 +1,7 @@ import { describe, it, before, after } from 'mocha'; import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; -import { execInContainer, pm2Registration } from '../framework/container.js'; +import { execInContainer, pm2Registration, AS_PM2_OPERATOR } from '../framework/container.js'; import { bootAndPeer } from '../framework/reconciler-suite.js'; import { waitFor } from '../framework/wait.js'; import { isDaemonUp } from '../framework/syncthing-real.js'; @@ -32,7 +32,7 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // request in the boot. const SHORT_MS = 1600; -const REQUEST_MARKER = '/root/.flux-pm2-kill-timeout-request'; +const REQUEST_MARKER = '/home/fluxuser/.flux-pm2-kill-timeout-request'; // How long a restart loop would take to show itself: pm2 restarts FluxOS // within seconds of being asked. const QUIET_MS = 45000; @@ -57,7 +57,7 @@ describe('a legacy node\'s request for a longer pm2 kill timeout never strands i // The registration as the multitool made it before it set a kill timeout, // with no request recorded for this boot: FluxOS asks as it restarts. const registerShort = async () => { - const r = await sh(`rm -f ${REQUEST_MARKER} && ${realPm2} restart flux --kill-timeout ${SHORT_MS} >/dev/null && ${realPm2} save >/dev/null`); + const r = await sh(`rm -f ${REQUEST_MARKER} && ${AS_PM2_OPERATOR} ${realPm2} restart flux --kill-timeout ${SHORT_MS} >/dev/null && ${AS_PM2_OPERATOR} ${realPm2} save >/dev/null`); expect(r.exitCode, `fixture: ${r.output}`).to.equal(0); }; diff --git a/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js b/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js index d4ee10cd22..0b2eae489a 100644 --- a/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js +++ b/test-infra/runner/tests/2209-a-single-writer-folder-type-has-one-writer.js @@ -393,7 +393,9 @@ describe('a single-writer folder type has one writer', function () { `rm -rf /tmp/s-${a.appName} && mkdir -p /tmp/s-${a.appName} && printf 'restored\\n' > /tmp/s-${a.appName}/restored.txt ` + `&& mkdir -p ${dir}/backup/local && tar -czf ${archive} -C /tmp/s-${a.appName} .`); expect(staged.exitCode, `fixture: staging the archive failed: ${staged.output}`).to.equal(0); - // The clear fails the way a volume gone read-only under an ext4 error does. + // The clear fails the way a volume gone read-only under an ext4 error does: the + // primary's data cannot be deleted. + await seedSyncScopedData(env, a.appName, a.primary); const lock = await execInContainer(client.container, `chattr +i ${dir}/appdata`); expect(lock.exitCode, `fixture: could not make appdata immutable: ${lock.output}`).to.equal(0); const from = client.getLastEventId(); diff --git a/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js b/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js index aeb5c992fc..ef8ee17a35 100644 --- a/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js +++ b/test-infra/runner/tests/2210-a-returning-node-waits-paused-for-its-role.js @@ -5,7 +5,7 @@ import { expect } from 'chai'; import { createTestEnv } from '../framework/test-env.js'; import { execInContainer, isAppContainerRunning, crashFluxos, releaseFluxos, shutdownFluxosGracefully, - redirectOutbound, clearOutboundRedirect, + redirectOutbound, clearOutboundRedirect, fluxosPid, } from '../framework/container.js'; import { pushTestApp } from '../framework/registry-helper.js'; import { buildSeedableApp } from '../framework/seed-helper.js'; @@ -218,8 +218,8 @@ describe('a returning node waits paused until its role is decided', function () this.timeout(300000); const since = await lastDaemonEventId(client(ARCANE)); const mark = client(ARCANE).getLastEventId(); - const pid = (await sh(ARCANE, 'cat /tmp/fluxos.pid')).stdout.trim(); - expect(pid, 'fixture: no FluxOS pid').to.match(/^\d+$/); + const pid = await fluxosPid(client(ARCANE).container); + expect(pid, 'fixture: no FluxOS pid').to.be.a('number'); // SIGTERM with no system shutdown under way: the service restarting. await sh(ARCANE, `kill -TERM ${pid}`); diff --git a/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js b/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js index 99dc139e7d..5ad0113a56 100644 --- a/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js +++ b/test-infra/runner/tests/2214-two-holders-that-see-each-other-never-both-seed.js @@ -5,7 +5,7 @@ import { getAppContainerStatus, blockPeerAccess, unblockPeerAccess, execInContainer, } from '../framework/container.js'; import { - setSyncState, setNoPeerData, resetSyncState, getFolderConfig, + resetSyncState, getFolderConfig, pinColdStart, pinHolding, } from '../framework/syncthing-control.js'; import { resetFdm, electMaster } from '../framework/fdm-control.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; @@ -43,37 +43,6 @@ async function isUp(client, appName) { return !!(status && status.status.startsWith('Up')); } -// A cold start on every holder: no index of its own and no connected peer holding -// the data, set before install so the first election sees it. -async function pinColdStart(holders, folder) { - await Promise.all(holders.map((i) => Promise.all([ - setSyncState({ - ip: subnet.nodeIp(i + 1), folder, state: 'idle', globalBytes: 0, inSyncBytes: 0, receiveOnlyChangedFiles: 0, - }), - setNoPeerData({ ip: subnet.nodeIp(i + 1), folder }), - ]))); -} - -// The owner's data on one holder, declared as entries, the way localHoldings reads it. -async function pinHolding(index, folder, bytes) { - const modified = new Date().toISOString(); - await setSyncState({ - ip: subnet.nodeIp(index + 1), - folder, - state: 'idle', - globalBytes: 0, - inSyncBytes: 0, - localChanged: [ - { - name: 'appdata', type: 'FILE_INFO_TYPE_DIRECTORY', size: 128, deleted: false, modified, - }, - { - name: 'appdata/seed-data', type: 'FILE_INFO_TYPE_FILE', size: bytes, deleted: false, modified, - }, - ], - }); - await setNoPeerData({ ip: subnet.nodeIp(index + 1), folder }); -} describe('two holders that can reach each other never both seed', function () { let env; @@ -199,9 +168,13 @@ describe('two holders that can reach each other never both seed', function () { it('seeds once, on the holder, when the holder has decided before the lowest address reads it', async function () { this.timeout(600000); - const app = await placeSplitApp('holderfirst', [[lowest, BEFORE_DECISION]]); - await heldAt(lowest, BEFORE_DECISION, app); + // Both held, so the lowest address is at its decision before the holder seeds: + // left to run, the holder seeds before the lowest address has confirmed its + // election, and the lowest address never reaches the decision at all. + const app = await placeSplitApp('holderfirst', contenders.map((i) => [i, BEFORE_DECISION])); + await Promise.all(contenders.map((i) => heldAt(i, BEFORE_DECISION, app))); + await release(holder, BEFORE_DECISION, app); await waitFor(async () => (await runners(app.name)).includes(holder), { timeout: 120000, interval: 1000, label: 'the holder seeds while the lowest address is held', }); diff --git a/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js b/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js index 5d84defbce..6b60712e2a 100644 --- a/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js +++ b/test-infra/runner/tests/2226-a-node-whose-api-port-changes-keeps-its-apps.js @@ -98,6 +98,13 @@ describe('a node whose api port changes keeps its apps', function () { idx = await seedAndInstall(env, app); nodeIp = subnet.nodeIp(idx + 1); await waitFor(isUp, { timeout: 120000, interval: 3000, label: 'the app running before the port moves' }); + // A move carries only the rows standing at the address it leaves: a peer that does + // not yet hold the app there refuses it. + const oldSocket = `${nodeIp}:${OLD_PORT}`; + const holdsAtOldPort = async (i) => (await dbClient(i + 1).getAppLocations(appName)).some((row) => row.ip === oldSocket); + await waitFor(async () => (await Promise.all([0, 1, 2].map(holdsAtOldPort))).every(Boolean), { + timeout: 120000, interval: 3000, label: 'every node holds the app at the old port', + }); }); after(async function () { diff --git a/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js b/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js index ac5f0f6b9b..eb5bd892c4 100644 --- a/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js +++ b/test-infra/runner/tests/2237-a-move-a-failure-or-a-crash-interrupted-is-finished-once.js @@ -8,8 +8,9 @@ import { configuredIpOf, crashFluxos, moveNodeAddress, releaseFluxos, } from '../framework/container.js'; import { - clearNodeAddress, clearRpcFailures, failRpc, getJournal, setNodeAddress, + clearNodeAddress, clearRpcFailures, failRpc, getJournal, holdRpc, releaseRpc, setNodeAddress, } from '../framework/daemon-control.js'; +import { askToBeDialed } from '../framework/peer-topology.js'; import { waitFor } from '../framework/wait.js'; import { bootAndPeer } from '../framework/reconciler-suite.js'; import { dbClient } from '../framework/db-client.js'; @@ -25,6 +26,11 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // Node 0 has its confirmation refused once; node 1 is crashed between // recording its move and acting on it. Node 2 is a bystander that holds the // events and reads the list. +// +// Node 1 runs no app, so a peer that hears its move once the chain lists it at +// the new address only has nothing to prove the move by, and drops it. Once it +// is back its confirmation is held, and it is asked to dial the bystander, until +// the bystander holds the move. const REFUSED = 0; const CRASHED = 1; @@ -39,6 +45,7 @@ describe('a move a failure or a crash interrupted is finished once', function () dumpLogsOnFailure(() => env); const moved = []; const failing = []; + const holding = []; const client = (i) => env.clients[i]; const socketOf = (ip) => (String(ip).includes(':') ? String(ip) : `${ip}:${API_PORT}`); const nodeList = async () => (await client(BYSTANDER).get(`/daemon/viewdeterministiczelnodelist?_=${Date.now()}`)).data || []; @@ -59,6 +66,17 @@ describe('a move a failure or a crash interrupted is finished once', function () return { from: socketOf(from), to: socketOf(to) }; } + // The move the bystander holds from node i's old address. + async function bystanderHolds(i, { from, to }) { + let held; + await waitFor(async () => { + held = (await dbClient(BYSTANDER + 1).getAppStateEvents({ ip: from, type: 'ipchanged' })) + .find((event) => event.data?.newIP === to); + return Boolean(held); + }, { timeout: 180000, interval: 5000, label: `the bystander holds node ${i}'s move` }); + return held; + } + // The move as the node recorded it, the move the bystander holds from its old // address, and where the list has it. async function expectFinishedOnce(i, { from, to }, pubkey) { @@ -68,12 +86,7 @@ describe('a move a failure or a crash interrupted is finished once', function () const record = await dbClient(i + 1).ownAddressRecord(); expect(record.socketAddress, `node ${i}'s recorded address`).to.equal(to); expect(record.recentIpChanges, `node ${i}'s IP changes counted`).to.have.lengthOf(1); - let held; - await waitFor(async () => { - held = (await dbClient(BYSTANDER + 1).getAppStateEvents({ ip: from, type: 'ipchanged' })) - .find((event) => event.data?.newIP === to); - return Boolean(held); - }, { timeout: 120000, interval: 5000, label: `the bystander holds node ${i}'s move` }); + const held = await bystanderHolds(i, { from, to }); expect(held.data.broadcastedAt, `the time node ${i}'s move is told with`).to.equal(record.movedAt); expect(await configuredIpOf(client(i).container), `node ${i}'s configured IP`).to.equal(to.split(':')[0]); } @@ -93,6 +106,7 @@ describe('a move a failure or a crash interrupted is finished once', function () this.timeout(60000); await Promise.all(moved.map((ip) => clearNodeAddress(ip).catch(() => {}))); await Promise.all(failing.map((ip) => clearRpcFailures(ip).catch(() => {}))); + await Promise.all(holding.map((ip) => releaseRpc(ip).catch(() => {}))); await env?.teardown(); }); @@ -125,8 +139,21 @@ describe('a move a failure or a crash interrupted is finished once', function () await node.waitForEvent('checkpoint:held', (d) => d.name === BEFORE_RECONCILE && d.key === addresses.to, DETECTION_MS, { afterId: anchor }); expect((await dbClient(CRASHED + 1).ownAddressRecord()).socketAddress, 'fixture: the move recorded before the crash').to.equal(addresses.to); await crashFluxos(node.container, { hold: true }); + // The daemon knows a node by the address its calls come from. + holding.push(toIp); + await holdRpc(toIp, 'createconfirmationtransaction'); + const bystander = client(BYSTANDER); + const peeredAnchor = bystander.getLastEventId(); await releaseFluxos(node.container); + // A node tells its move to each peer as it connects. + await waitFor(async () => askToBeDialed(bystander, node, API_PORT).then(() => true, () => false), { + timeout: 180000, interval: 5000, label: `node ${CRASHED} takes the bystander's ask to be dialed`, + }); + await bystander.waitForEvent('peers:added', (d) => d.ip === toIp, 120000, { afterId: peeredAnchor }); + await bystanderHolds(CRASHED, addresses); + expect(await listedAt(pubkey), 'fixture: the bystander heard the move with the chain listing the old address').to.equal(addresses.from); + await releaseRpc(toIp); await expectFinishedOnce(CRASHED, addresses, pubkey); }); }); diff --git a/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js b/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js new file mode 100644 index 0000000000..d89c0c8b5a --- /dev/null +++ b/test-infra/runner/tests/2241-an-apps-g-components-run-together-on-its-primary.js @@ -0,0 +1,110 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, +} from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, on real syncthing daemons: its +// primary runs both, and every other holder runs neither. Every wait holds that no +// two nodes run any of its g: components at once (g-components.js). + +const APP_UID = 1000; + +describe('an app\'s g: components run together on its primary', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2etwin${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // Placed in this order, so the election ranks them in it; a third node holds + // nothing and keeps the ring whole. + const FIRST = 0; + const SECOND = 1; + const HOLDERS = [FIRST, SECOND]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (component) => (await getFolderConfig(client(i), folderOf(component)))?.type)); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'twin', { user: APP_UID }); + const app = await buildSeedableApp({ + env, + name: appName, + instances: HOLDERS.length, + compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + + await installOnNodes(env, app, [FIRST]); + await waitHolding(async () => (await oneHolderAtMost())[FIRST].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + await installOnNodes(env, app, [SECOND]); + await Promise.all(COMPONENTS.map((component) => waitFor(() => isFolderSynced(client(SECOND), folderOf(component)), { + timeout: 300000, interval: 3000, label: `the second holder has the primary's ${component} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs every g: component on the primary, and none on the standby', async function () { + this.timeout(180000); + await holdsFor(60000, { [FIRST]: COMPONENTS, [SECOND]: [] }); + expect(await folderTypes(FIRST), 'the primary\'s folders').to.deep.equal(['sendreceive', 'sendreceive']); + expect(await folderTypes(SECOND), 'the standby\'s folders').to.deep.equal(['receiveonly', 'receiveonly']); + }); + + it('moves every g: component together when the primary dies', async function () { + this.timeout(420000); + // The whole machine: the process, the containers and the daemon. + await crashFluxos(client(FIRST).container, { hold: true }); + await Promise.all(COMPONENTS.map((component) => execInContainer(client(FIRST).container, `docker kill ${folderOf(component)}`))); + await stopDaemon(client(FIRST)); + + await waitHolding(async () => (await oneHolderAtMost())[SECOND].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the second holder runs every g: component', + }); + await waitFor(async () => (await folderTypes(SECOND)).every((type) => type === 'sendreceive'), { + timeout: 120000, interval: 2000, label: 'every folder of the new primary sends', + }); + }); + + it('brings the old primary back as a standby of the whole app', async function () { + this.timeout(420000); + await startDaemon(client(FIRST), { paused: true }); + await releaseFluxos(client(FIRST).container); + + await waitFor(async () => (await folderTypes(FIRST)).every((type) => type === 'receiveonly'), { + timeout: 300000, interval: 3000, label: 'every folder of the old primary receives', + }); + await holdsFor(60000, { [FIRST]: [], [SECOND]: COMPONENTS }); + }); +}); diff --git a/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js new file mode 100644 index 0000000000..700a8f7e01 --- /dev/null +++ b/test-infra/runner/tests/2242-an-update-that-adds-a-g-component-runs-the-app-on-one-node.js @@ -0,0 +1,121 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { allocatePortFor } from '../framework/port-allocator.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A specification update gives a running app a second single-writer (g:) +// component, on real syncthing daemons. +// +// A change to the components is a structure change: each holder removes the app +// and installs it at the new specification, on its own reinstall pass. Between the +// two passes one holder is at the new specification and the other still runs the +// old one, and after both the app starts afresh. Through all of it no two nodes run +// any g: component of the app at once (g-components.js), and once settled one node +// runs both and the other runs neither. +// +// DRIVEN BY BLOCKS: the reinstall pass fires on +// `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). + +const APP_UID = 1000; + +describe('an update that adds a g: component runs the app on one node', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eaddg${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const HOLDERS = [0, 1]; + let app; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + // The specification a holder has installed, by its hash. + const installedHash = async (i) => { + const res = await client(i).getInstalledApps(); + return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; + }; + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'addg', { user: APP_UID }); + app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: [gComponent(appName, 'db')], + }); + await installOnNodes(env, app, [HOLDERS[0]]); + await waitHolding(async () => (await oneHolderAtMost())[HOLDERS[0]].includes('db'), { + timeout: 300000, interval: 3000, label: 'the first holder runs db', + }); + await installOnNodes(env, app, [HOLDERS[1]]); + await waitFor(() => isFolderSynced(client(HOLDERS[1]), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the second holder has the primary\'s db data', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('never runs the app on two nodes, and settles with both g: components on one', async function () { + // A backstop: the block drive bounds this test (suite 85). + this.timeout(1800000); + const updated = await buildSeedableUpdate(app, (spec) => { + // The new component's port, allocated as registration allocates db's. + spec.compose.push({ ...gComponent(appName, 'web'), ports: [allocatePortFor(appName, 1)] }); + }); + await seedSpecUpdate(env, updated, HOLDERS); + await stopTicker(); + + let breach = null; + let settled = null; + // Until both holders have reinstalled: each does on its own pass. + await driveUntil(client(HOLDERS[0]), async () => { + try { + const running = await oneHolderAtMost(); + settled = HOLDERS.find((i) => running[i].length === COMPONENTS.length) ?? null; + const hashes = await Promise.all(HOLDERS.map(installedHash)); + return settled !== null && hashes.every((hash) => hash === updated.hash); + } catch (error) { + if (error?.name !== 'AssertionError') throw error; + breach = error; + return true; + } + }, { blocks: 216, label: 'both holders are at the new specification and one runs db and web' }); + if (breach) throw breach; + + const other = HOLDERS.find((i) => i !== settled); + expect((await getFolderConfig(client(settled), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + expect(running[settled], 'the primary').to.deep.equal(COMPONENTS); + return isFolderSynced(client(other), folderOf('web')); + }, { timeout: 300000, interval: 3000, label: 'the standby has the primary\'s web data' }); + expect((await getFolderConfig(client(other), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + expect((await oneHolderAtMost())[other], 'the standby').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js b/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js new file mode 100644 index 0000000000..793c9c4f87 --- /dev/null +++ b/test-infra/runner/tests/2243-a-rolling-upgrade-standby-first-brings-up-every-g-component.js @@ -0,0 +1,115 @@ +// base-image: FLUX_E2E_BASE_TAG +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, held by two nodes on a release +// that starts only the first, upgraded to this tree one node at a time: the +// standby first, then the primary. Real syncthing daemons. +// +// On the release the primary runs the first g: component and never the second. +// With the standby upgraded and the primary not, nothing changes: the upgraded +// standby stands aside for a primary that holds the app, and starts nothing. With +// the primary upgraded too it starts the second component beside the first. The +// primary's address sorts after the standby's, so the second component's folder is +// seeded by the node that writes the app, not by the address order. At no point +// does any g: component of the app run on two nodes. +// +// Node 2 runs this tree throughout and holds nothing: the third node of the ring. +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + +const APP_UID = 1000; +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; + +describe('a rolling upgrade, standby first, brings up every g: component', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eroll${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const STANDBY = 0; + const PRIMARY = 1; + const RING = 2; + const HOLDERS = [STANDBY, PRIMARY]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS the nodes upgrade from: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + nodeImageTags: { [STANDBY]: BASE_TAG, [PRIMARY]: BASE_TAG }, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + if (BASE_TAG === env.nodeImageTag(RING)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG: the suite upgrades from the one to the other'); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + expect(`${client(STANDBY).ip}:16127` < `${client(PRIMARY).ip}:16127`, 'fixture: the standby sorts before the primary by address').to.equal(true); + + await pushTestApp(appName, 'v1', 'roll', { user: APP_UID }); + const app = await buildSeedableApp({ + env, + name: appName, + instances: HOLDERS.length, + compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs only the first g: component on the release', async function () { + this.timeout(180000); + await holdsFor(60000, { [STANDBY]: [], [PRIMARY]: ['db'] }); + }); + + it('starts nothing on the upgraded standby while the primary on the release holds the app', async function () { + this.timeout(420000); + await env.upgradeNode(STANDBY, { tag: env.nodeImageTag(RING) }); + await holdsFor(90000, { [STANDBY]: [], [PRIMARY]: ['db'] }); + }); + + it('starts every g: component on the primary once it is upgraded', async function () { + this.timeout(600000); + await env.upgradeNode(PRIMARY, { tag: env.nodeImageTag(RING) }); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 420000, interval: 3000, label: 'the primary runs db and web', + }); + expect((await getFolderConfig(client(PRIMARY), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitHolding(async () => { + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + return isFolderSynced(client(STANDBY), folderOf('web')); + }, { timeout: 300000, interval: 3000, label: 'the standby has the primary\'s web data' }); + expect((await getFolderConfig(client(STANDBY), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + }); +}); diff --git a/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js b/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js new file mode 100644 index 0000000000..2d9843495a --- /dev/null +++ b/test-infra/runner/tests/2244-a-rolling-upgrade-primary-first-brings-up-every-g-component.js @@ -0,0 +1,115 @@ +// base-image: FLUX_E2E_BASE_TAG +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, held by two nodes on a release +// that starts only the first, upgraded to this tree one node at a time: the +// primary first, then the standby. Real syncthing daemons. +// +// On the release the primary runs the first g: component and never the second, +// and the standby, whose address sorts first, takes the second component's empty +// folder as its own seed. Upgraded, the primary starts the second component beside +// the first while the standby is still on the release: the folder is seeded by the +// node that writes the app, whatever a standby on the release decided about it. +// Upgrading the standby changes nothing. At no point does any g: component of the +// app run on two nodes. +// +// Node 2 runs this tree throughout and holds nothing: the third node of the ring. +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + +const APP_UID = 1000; +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; + +describe('a rolling upgrade, primary first, brings up every g: component', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2erollp${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const STANDBY = 0; + const PRIMARY = 1; + const RING = 2; + const HOLDERS = [STANDBY, PRIMARY]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS the nodes upgrade from: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + nodeImageTags: { [STANDBY]: BASE_TAG, [PRIMARY]: BASE_TAG }, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + if (BASE_TAG === env.nodeImageTag(RING)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG: the suite upgrades from the one to the other'); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + expect(`${client(STANDBY).ip}:16127` < `${client(PRIMARY).ip}:16127`, 'fixture: the standby sorts before the primary by address').to.equal(true); + + await pushTestApp(appName, 'v1', 'roll', { user: APP_UID }); + const app = await buildSeedableApp({ + env, + name: appName, + instances: HOLDERS.length, + compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs only the first g: component on the release', async function () { + this.timeout(180000); + await holdsFor(60000, { [STANDBY]: [], [PRIMARY]: ['db'] }); + }); + + it('starts every g: component on the primary once it is upgraded, with the standby on the release', async function () { + this.timeout(600000); + await env.upgradeNode(PRIMARY, { tag: env.nodeImageTag(RING) }); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 420000, interval: 3000, label: 'the primary runs db and web', + }); + expect((await getFolderConfig(client(PRIMARY), folderOf('web')))?.type, 'the primary\'s web folder').to.equal('sendreceive'); + await waitHolding(async () => { + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + return isFolderSynced(client(STANDBY), folderOf('web')); + }, { timeout: 300000, interval: 3000, label: 'the standby on the release has the primary\'s web data' }); + }); + + it('changes nothing when the standby is upgraded', async function () { + this.timeout(420000); + await env.upgradeNode(STANDBY, { tag: env.nodeImageTag(RING) }); + await holdsFor(90000, { [STANDBY]: [], [PRIMARY]: COMPONENTS }); + expect((await getFolderConfig(client(STANDBY), folderOf('web')))?.type, 'the standby\'s web folder').to.equal('receiveonly'); + }); +}); diff --git a/test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js b/test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js new file mode 100644 index 0000000000..a4e5b4b8d6 --- /dev/null +++ b/test-infra/runner/tests/2245-a-g-component-that-dies-on-its-primary-is-restarted-there.js @@ -0,0 +1,111 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, on real syncthing daemons. One of +// them dies on the primary while its FluxOS, its syncthing and the other component +// go on running. +// +// The primary still holds the app: it runs the other component, and its folders +// still send. It starts the dead one again where it was. A standby that started it +// would run the app on two nodes - the one it started, beside the one the primary +// still runs - so no other node starts any of its g: components meanwhile. + +const APP_UID = 1000; + +describe('a g: component that dies on its primary is restarted there', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egdies${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // Placed in this order, so the election ranks them in it; a third node holds + // nothing and keeps the ring whole. + const PRIMARY = 0; + const STANDBY = 1; + const HOLDERS = [PRIMARY, STANDBY]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + // When docker last started a component's container on a node. + const startedAt = async (i, name) => { + const r = await execInContainer(client(i).container, `docker inspect -f '{{.State.StartedAt}}' ${folderOf(name)}`); + if (r.exitCode !== 0 || !r.stdout.trim()) throw new Error(`docker inspect ${folderOf(name)} on node ${i} failed: ${r.stderr || r.stdout}`); + return r.stdout.trim(); + }; + // Kills the given components on the primary, and holds until it runs every g: + // component again, each killed one in a container docker started after the kill. + const restartedInPlace = async (names, label) => { + const before = await Promise.all(names.map((name) => startedAt(PRIMARY, name))); + await Promise.all(names.map((name) => execInContainer(client(PRIMARY).container, `docker kill ${folderOf(name)}`))); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + if (running[PRIMARY].length !== COMPONENTS.length) return false; + const after = await Promise.all(names.map((name) => startedAt(PRIMARY, name))); + return after.every((at, k) => at !== before[k]); + }, { timeout: 300000, interval: 2000, label: `${label}: the primary runs every g: component again, the killed ones restarted` }); + await holdsFor(60000, { [PRIMARY]: COMPONENTS, [STANDBY]: [] }); + expect(await Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(PRIMARY), folderOf(name)))?.type)), 'the primary\'s folders') + .to.deep.equal(['sendreceive', 'sendreceive']); + }; + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gdies', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the primary runs every g: component', + }); + await installOnNodes(env, app, [STANDBY]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(STANDBY), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the standby has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('restarts the second g: component on the primary when it dies there', async function () { + this.timeout(600000); + await restartedInPlace(['web'], 'web killed'); + }); + + it('restarts the first g: component on the primary when it dies there', async function () { + this.timeout(600000); + await restartedInPlace(['db'], 'db killed'); + }); + + it('restarts both on the primary when both die there and its FluxOS goes on running', async function () { + this.timeout(600000); + await restartedInPlace(COMPONENTS, 'db and web killed'); + }); +}); diff --git a/test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js b/test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js new file mode 100644 index 0000000000..456b3f434a --- /dev/null +++ b/test-infra/runner/tests/2246-a-silent-primary-is-not-started-beside-while-any-folder-is-connected.js @@ -0,0 +1,121 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, getDeviceId, getRemoteState, setFolderPaused, stopDaemon, +} from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app with two single-writer (g:) components, on real syncthing daemons. Its +// primary's FluxOS stops answering while its containers and its syncthing go on +// running, with one of the app's folders paused there and the other still shared. +// +// A standby reads a silent primary as gone only when its own syncthing shows the +// primary's connection gone for EVERY folder of the app. A folder still connected +// is a component the primary may still be writing, so the standby starts nothing: +// started, the app would run on both nodes. Once the whole machine is down - every +// folder's connection gone - the standby takes the app over. + +const APP_UID = 1000; +const HELD_PASSES = 3; + +describe('a silent primary is not started beside while any of its folders is connected', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egsilent${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // The anchor: the first g: component, which keys the app's election records. + const identifier = `db_${appName}`; + const PRIMARY = 0; + const STANDBY = 1; + const HOLDERS = [PRIMARY, STANDBY]; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const silenceVerdicts = (verdict) => client(STANDBY).getDecisionCount('peer:silenceVerdict', identifier, verdict); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gsilent', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the primary runs every g: component', + }); + await installOnNodes(env, app, [STANDBY]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(STANDBY), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the standby has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(120000); + await releaseFluxos(client(PRIMARY).container).catch(() => {}); + await env?.teardown(); + }); + + it('starts nothing beside a silent primary whose second folder is still connected', async function () { + this.timeout(600000); + await crashFluxos(client(PRIMARY).container, { hold: true }); + await setFolderPaused(client(PRIMARY), folderOf('db'), true); + + const primaryDevice = await getDeviceId(client(PRIMARY)); + await waitFor(async () => (await getRemoteState(client(STANDBY), folderOf('db'), primaryDevice)) !== 'valid', { + timeout: 120000, interval: 2000, label: 'the standby\'s syncthing no longer sees the primary sharing db', + }); + expect(await getRemoteState(client(STANDBY), folderOf('web'), primaryDevice), 'fixture: the standby\'s syncthing sees the primary sharing web').to.equal('valid'); + + // Each is a pass in which the standby asked the silent primary and judged its + // silence: alive, by the folder still connected. + const gone = await silenceVerdicts('gone'); + const from = await silenceVerdicts('connectionAlive'); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + expect(running[PRIMARY], 'the silent primary').to.deep.equal(COMPONENTS); + return await silenceVerdicts('connectionAlive') >= from + HELD_PASSES; + }, { timeout: 300000, interval: 3000, label: `${HELD_PASSES} passes in which the standby judged the silent primary alive` }); + expect(await silenceVerdicts('gone') - gone, 'passes that read the silent primary as gone').to.equal(0); + expect((await oneHolderAtMost())[STANDBY], 'the standby').to.deep.equal([]); + }); + + it('takes the app over once the primary\'s whole machine is down', async function () { + this.timeout(600000); + const gone = await silenceVerdicts('gone'); + await Promise.all(COMPONENTS.map((name) => execInContainer(client(PRIMARY).container, `docker kill ${folderOf(name)}`))); + await stopDaemon(client(PRIMARY)); + + await waitHolding(async () => (await oneHolderAtMost())[STANDBY].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the standby runs every g: component', + }); + expect(await silenceVerdicts('gone') - gone, 'passes that read the primary as gone').to.be.above(0); + await waitFor(async () => (await Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(STANDBY), folderOf(name)))?.type))) + .every((type) => type === 'sendreceive'), { + timeout: 120000, interval: 2000, label: 'every folder of the new primary sends', + }); + }); +}); diff --git a/test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js b/test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js new file mode 100644 index 0000000000..f107c1aa1e --- /dev/null +++ b/test-infra/runner/tests/2247-an-owner-stop-and-start-keeps-every-g-component-on-the-primary.js @@ -0,0 +1,179 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, electionIndexOf } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding, electionDecisionCount } from '../framework/wait.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, +} from '../framework/syncthing-real.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An owner stops and starts an app with two single-writer (g:) components, on real +// syncthing daemons: the whole app on every node at once, and one component alone +// on its primary. +// +// Stopped and started everywhere, the app comes back on the node that was its +// primary, every g: component of it, not on whichever node the election order +// would pick: the primary is put below index 0 first, as in suite 2215. One +// component stopped on the primary stays down by its lock while the primary goes +// on running the other, and no other node starts either - the primary still holds +// the app. Started again, it comes back on the primary. + +const APP_UID = 1000; +const HELD_PASSES = 3; + +describe('an owner stop and start keeps every g: component on the primary', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egowner${Date.now()}`; + const COMPONENTS = ['db', 'web']; + // The anchor: the first g: component, which keys the app's election records. + const identifier = `db_${appName}`; + // Placed in this order, so the election ranks them in it. A fourth node holds + // nothing and keeps the discovery ring whole. + const FIRST = 0; + const HOLDERS = [0, 1, 2]; + let PRIMARY; + let STANDBYS; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const runners = async () => Object.entries(await oneHolderAtMost()) + .filter(([, running]) => running.length).map(([i]) => Number(i)); + const ownerAuth = async (i) => (await authenticate(client(i).url, appOwnerKey())).zelidauth; + const owner = async (command, target, via, { global = false } = {}) => { + const res = await client(via).getAuthed(`/apps/${command}/${target}${global ? '/true' : ''}`, await ownerAuth(via)); + expect(res?.status, `owner ${global ? 'global ' : ''}${command} of ${target} through node ${via}: ${JSON.stringify(res)}`).to.equal('success'); + }; + const evaluated = (nodes) => Promise.all(nodes.map((i) => electionDecisionCount(client(i), identifier, 'evaluated'))); + const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(i), folderOf(name)))?.type)); + + // What every holder runs, asserted on every poll, until each of `nodes` has run + // HELD_PASSES election passes over the app. + const holdsThroughPasses = async (nodes, expected, label) => { + const from = await evaluated(nodes); + await waitHolding(async () => { + expect(await oneHolderAtMost(), `${label}: what each holder runs`).to.deep.equal(expected); + const now = await evaluated(nodes); + return now.every((n, k) => n >= from[k] + HELD_PASSES); + }, { timeout: 300000, interval: 2000, label: `${label}: ${HELD_PASSES} election passes on nodes ${nodes.join(', ')}` }); + }; + const nothingRuns = () => Object.fromEntries(HOLDERS.map((i) => [i, []])); + const onPrimary = (running) => ({ ...nothingRuns(), [PRIMARY]: running }); + + before(async function () { + this.timeout(1500000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gowner', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + + // One at a time: a node learns of the app only when it is installed, so the + // order is the placement order. + await installOnNodes(env, app, [FIRST]); + await waitHolding(async () => (await oneHolderAtMost())[FIRST].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + for (const i of HOLDERS.filter((h) => h !== FIRST)) { + // eslint-disable-next-line no-await-in-loop + await installOnNodes(env, app, [i]); + // eslint-disable-next-line no-await-in-loop + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(i), folderOf(name)), { + timeout: 300000, interval: 3000, label: `holder ${i} has the first holder's ${name} data`, + }))); + } + + // The first holder dies whole - process, containers and daemon - and another + // takes the app over; then it comes back, as a standby still ranked ahead of + // the primary. + await crashFluxos(client(FIRST).container, { hold: true }); + await Promise.all(COMPONENTS.map((name) => execInContainer(client(FIRST).container, `docker kill ${folderOf(name)}`))); + await stopDaemon(client(FIRST)); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + return HOLDERS.some((i) => i !== FIRST && running[i].length === COMPONENTS.length); + }, { timeout: 300000, interval: 3000, label: 'another holder runs every g: component' }); + [PRIMARY] = await runners(); + STANDBYS = HOLDERS.filter((i) => i !== PRIMARY); + + await startDaemon(client(FIRST)); + await releaseFluxos(client(FIRST).container); + await waitHolding(async () => { + await oneHolderAtMost(); + return (await folderTypes(FIRST)).every((type) => type === 'receiveonly'); + }, { timeout: 300000, interval: 2000, label: 'the first holder is back as a standby of the whole app' }); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(FIRST), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the first holder has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs the app on a primary the election order ranks below another holder', async function () { + this.timeout(180000); + await holdsFor(30000, onPrimary(COMPONENTS)); + expect(await electionIndexOf(env, appName, FIRST), 'the first holder\'s place').to.equal(0); + expect(await electionIndexOf(env, appName, PRIMARY), 'the primary\'s place').to.be.above(0); + }); + + it('brings every g: component back on its primary when its owner stops the app everywhere and starts it everywhere', async function () { + this.timeout(900000); + await owner('appstop', appName, PRIMARY, { global: true }); + await waitHolding(async () => (await runners()).length === 0, { + timeout: 180000, interval: 2000, label: 'the app stops everywhere', + }); + await holdsThroughPasses(STANDBYS, nothingRuns(), 'stopped everywhere'); + + await owner('appstart', appName, FIRST, { global: true }); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 2000, label: 'started everywhere: the primary runs every g: component again', + }); + await holdsThroughPasses(STANDBYS, onPrimary(COMPONENTS), 'started everywhere'); + expect(await folderTypes(PRIMARY), 'the primary\'s folders').to.deep.equal(['sendreceive', 'sendreceive']); + }); + + for (const [stopped, kept] of [['web', 'db'], ['db', 'web']]) { + it(`keeps ${kept} on the primary and runs ${stopped} nowhere when its owner stops ${stopped} alone there, and brings it back there once started`, async function () { + this.timeout(900000); + await owner('appstop', `${stopped}_${appName}`, PRIMARY); + await waitHolding(async () => { + const running = await oneHolderAtMost(); + return running[PRIMARY].length === 1 && running[PRIMARY][0] === kept; + }, { timeout: 180000, interval: 2000, label: `${stopped} stops on the primary` }); + await holdsThroughPasses(HOLDERS, onPrimary([kept]), `${stopped} stopped on the primary`); + + await owner('appstart', `${stopped}_${appName}`, PRIMARY); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].length === COMPONENTS.length, { + timeout: 300000, interval: 2000, label: `${stopped} started on the primary: it runs every g: component again`, + }); + await holdsThroughPasses(STANDBYS, onPrimary(COMPONENTS), `${stopped} started again`); + }); + } +}); diff --git a/test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js b/test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js new file mode 100644 index 0000000000..c040091e01 --- /dev/null +++ b/test-infra/runner/tests/2248-an-update-that-moves-the-first-g-component-runs-the-app-on-one-node.js @@ -0,0 +1,129 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Specification updates move the first single-writer (g:) component of a running +// app with two, on real syncthing daemons: one reverses the components, the next +// removes the one that became first. +// +// The first g: component names the app to the election - it keys the election's +// records, and it is the component a node on a release asks a peer about - so each +// update moves the app's election onto another component. Each holder reinstalls +// on its own pass (suite 2242); through all of it no two nodes run any g: +// component of the app at once (g-components.js), and once settled one node runs +// every g: component the app has and the other runs none. +// +// DRIVEN BY BLOCKS: the reinstall pass fires on +// `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). + +const APP_UID = 1000; + +describe('an update that moves the first g: component runs the app on one node', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2emoveg${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const HOLDERS = [0, 1]; + let app; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + // The specification a holder has installed, by its hash. + const installedHash = async (i) => { + const res = await client(i).getInstalledApps(); + return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; + }; + + // Seeds the update and drives blocks until both holders are at it and one runs + // exactly `expected`, then holds that. Returns the node that runs it. + const settlesOnOneNode = async (updated, expected) => { + await seedSpecUpdate(env, updated, HOLDERS); + await stopTicker(); + + let breach = null; + let settled = null; + await driveUntil(client(HOLDERS[0]), async () => { + try { + const running = await oneHolderAtMost(); + settled = HOLDERS.find((i) => running[i].length) ?? null; + const hashes = await Promise.all(HOLDERS.map(installedHash)); + return settled !== null && JSON.stringify(running[settled]) === JSON.stringify(expected) + && hashes.every((hash) => hash === updated.hash); + } catch (error) { + if (error?.name !== 'AssertionError') throw error; + breach = error; + return true; + } + }, { blocks: 216, label: `both holders are at the new specification and one runs [${expected}]` }); + if (breach) throw breach; + + const other = HOLDERS.find((i) => i !== settled); + await holdsFor(60000, { [settled]: expected, [other]: [] }); + expect(await Promise.all(expected.map(async (name) => (await getFolderConfig(client(settled), folderOf(name)))?.type)), 'the primary\'s folders') + .to.deep.equal(expected.map(() => 'sendreceive')); + return settled; + }; + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'moveg', { user: APP_UID }); + app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [HOLDERS[0]]); + await waitHolding(async () => (await oneHolderAtMost())[HOLDERS[0]].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + await installOnNodes(env, app, [HOLDERS[1]]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(HOLDERS[1]), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the second holder has the primary's ${name} data`, + }))); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs the app on one node once an update puts its second g: component first', async function () { + // A backstop: the block drive bounds this test (suite 85). + this.timeout(1800000); + // Each component keeps the port it was registered with. + app = await buildSeedableUpdate(app, (spec) => { spec.compose.reverse(); }); + expect(app.spec.compose.map((c) => c.name), 'fixture: the update\'s components').to.deep.equal(['web', 'db']); + await settlesOnOneNode(app, COMPONENTS); + }); + + it('runs the app on one node once an update removes the g: component that is first', async function () { + this.timeout(1800000); + app = await buildSeedableUpdate(app, (spec) => { spec.compose = spec.compose.filter((c) => c.name !== 'web'); }); + expect(app.spec.compose.map((c) => c.name), 'fixture: the update\'s components').to.deep.equal(['db']); + await settlesOnOneNode(app, ['db']); + }); +}); diff --git a/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js b/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js new file mode 100644 index 0000000000..d7f4737205 --- /dev/null +++ b/test-infra/runner/tests/2249-a-surplus-trim-keeps-the-copy-running-any-g-component.js @@ -0,0 +1,196 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, electionOrder } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding, waitForGiveUpConsidered } from '../framework/wait.js'; +import { advanceBlock, driveUntil, startTicker, stopTicker } from '../framework/daemon-control.js'; +import { + isDaemonUp, isFolderSynced, getFolderConfig, stopDaemon, startDaemon, +} from '../framework/syncthing-real.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Trimming a surplus copy of an app with two single-writer (g:) components, when +// the copy the surplus rule picks - the newest - is the app's primary, running its +// second g: component with the first stopped there by its owner. Real syncthing +// daemons. +// +// The newest copy stands aside unless it runs the app's writer, and every g: +// component is the writer: the primary runs them together. So the newest copy +// stays while it runs ANY of them, and the next copy trims instead, on the +// newest's own word that it holds the app (suite 96). +// +// THE SHAPE: the first holder places and seeds the app, the second takes it over +// when the first dies, and the first comes back as a standby - so the primary is +// the newest copy. A stub peer then announces the app, backdated so it is the most +// senior holder: three holders against an instance count of two. + +const APP_UID = 1000; +const OLDER = 0; +const NEWEST = 1; +const RING = 2; +const STUB_INDEX = 3; +const HOLDERS = [OLDER, NEWEST]; + +describe('a surplus trim keeps the copy running any g: component', function () { + let env; + let held; + dumpLogsOnFailure(() => env); + + const appName = `e2egsurplus${Date.now()}`; + const COMPONENTS = ['db', 'web']; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const ownerAuth = async (i) => (await authenticate(client(i).url, appOwnerKey())).zelidauth; + const folderTypes = async (i) => Promise.all(COMPONENTS.map(async (name) => (await getFolderConfig(client(i), folderOf(name)))?.type)); + // Read from a holder's own list, senior first. + const order = async (via) => { + const res = await client(via).getAppLocations(appName); + if (res?.status !== 'success' || !Array.isArray(res.data) || !res.data.length) { + throw new Error(`app locations unreadable: ${JSON.stringify(res)?.slice(0, 200)}`); + } + return electionOrder(res.data).map((location) => location.ip.split(':')[0]); + }; + // Whether a holder has the app installed, from an answer it gave: a read that + // fails is asked again, never taken as the app gone. + const installedOn = async (i) => { + let installed = null; + await waitFor(async () => { + const res = await client(i).getInstalledApps(); + if (res?.status !== 'success' || !Array.isArray(res.data)) throw new Error(`installed apps unreadable on node ${i}: ${JSON.stringify(res)?.slice(0, 200)}`); + installed = res.data.some((a) => a.name === appName); + return true; + }, { timeout: 60000, interval: 2000, label: `node ${i} answers what it has installed` }); + return installed; + }; + // What the decision reads, asked the way the product asks it: the newest copy's + // /apps/heldcomponents, from the next copy. + const newestAnswersHeld = async () => { + const res = await execInContainer(client(OLDER).container, `curl -sf -m 5 http://${client(NEWEST).ip}:16127/apps/heldcomponents`); + return res.exitCode === 0 && res.output.includes(folderOf('web')); + }; + + before(async function () { + this.timeout(1500000); + env = await createTestEnv({ + hookCtx: this, + nodes: STUB_INDEX + 1, + stubPeers: [STUB_INDEX], + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all([OLDER, NEWEST, RING].map((i) => waitFor(() => isDaemonUp(client(i)), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'gsurplus', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await installOnNodes(env, app, [OLDER]); + await waitHolding(async () => (await oneHolderAtMost())[OLDER].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the first holder runs every g: component', + }); + await installOnNodes(env, app, [NEWEST]); + await Promise.all(COMPONENTS.map((name) => waitFor(() => isFolderSynced(client(NEWEST), folderOf(name)), { + timeout: 300000, interval: 3000, label: `the second holder has the first holder's ${name} data`, + }))); + + // The first holder dies whole, the second takes the app over, and the first + // comes back as its standby. + await crashFluxos(client(OLDER).container, { hold: true }); + await Promise.all(COMPONENTS.map((name) => execInContainer(client(OLDER).container, `docker kill ${folderOf(name)}`))); + await stopDaemon(client(OLDER)); + await waitHolding(async () => (await oneHolderAtMost())[NEWEST].length === COMPONENTS.length, { + timeout: 300000, interval: 3000, label: 'the second holder runs every g: component', + }); + await startDaemon(client(OLDER), { paused: true }); + await releaseFluxos(client(OLDER).container); + await waitHolding(async () => { + await oneHolderAtMost(); + return (await folderTypes(OLDER)).every((type) => type === 'receiveonly'); + }, { timeout: 300000, interval: 2000, label: 'the first holder is back as a standby of the whole app' }); + + // Its owner stops the first g: component alone on the primary. + const res = await client(NEWEST).getAuthed(`/apps/appstop/db_${appName}`, await ownerAuth(NEWEST)); + expect(res?.status, `owner stop of db on the primary: ${JSON.stringify(res)}`).to.equal('success'); + await waitHolding(async () => JSON.stringify((await oneHolderAtMost())[NEWEST]) === JSON.stringify(['web']), { + timeout: 180000, interval: 2000, label: 'the primary runs web alone', + }); + expect(await order(OLDER), 'fixture: the primary is the newest copy').to.deep.equal([client(OLDER).ip, client(NEWEST).ip]); + await waitFor(newestAnswersHeld, { + timeout: 180000, interval: 3000, label: 'the newest copy answers heldcomponents with web, asked from the next copy', + }); + + // The surplus has to survive until the test drives the trim pass, which runs + // on block heights: every node is brought level with a frozen tip first, so + // none processes its way onto one while the fixture is assembling. + await stopTicker(); + const real = [OLDER, NEWEST, RING]; + const seen = real.map((i) => client(i).getLastEventId()); + const { currentHeight } = await advanceBlock(); + await Promise.all(real.map((i, k) => client(i).waitForEvent( + 'block:processed', (d) => d.height >= currentHeight, 120000, { afterId: seen[k] }, + ))); + + const stub = env.stubPeerClients.get(STUB_INDEX); + await waitFor(async () => (await stub.connectedNodes()) > 0, { + timeout: 120000, interval: 2000, label: 'the fleet has connected to the stub peer', + }); + held = stub.holdApp(appName, { hash: app.hash, runningSince: Date.now() - 24 * 60 * 60 * 1000 }); + await held.started; + await Promise.all(HOLDERS.map((i) => waitFor(async () => (await order(i)).length === 3, { + timeout: 120000, interval: 2000, label: `holder ${i} counts the stub as a third holder`, + }))); + }); + + after(async function () { + this.timeout(120000); + held?.stop(); + await startTicker().catch(() => {}); + await env?.teardown(); + }); + + it('keeps the newest copy, which runs the second g: component with the first stopped, and trims the next copy instead', async function () { + this.timeout(900000); + let kept = null; + const keeping = waitForGiveUpConsidered( + client(NEWEST), appName, + (d) => d.giveUp === false && d.reason === 'SURPLUS', + 600000, + { afterId: client(NEWEST).getLastEventId() }, + ).then((entry) => { kept = entry.data; return entry; }); + keeping.catch(() => {}); + + // The pass runs every removeFluxAppsPeriod * speedMultiplier blocks and this + // drive spends about one period, so there is one attempt: a block is driven + // only while the newest copy is answering what the next copy will ask it. + await driveUntil(client(OLDER), async () => { + if (!await installedOn(NEWEST)) return true; + if (!await installedOn(OLDER)) return kept !== null; + await waitFor(newestAnswersHeld, { + timeout: 120000, interval: 3000, label: 'the newest copy still answers heldcomponents before another block is driven', + }); + return false; + }, { timeoutMs: 600000, label: 'a copy is trimmed and the newest has declined to stand aside' }); + await startTicker(); + + expect(await installedOn(NEWEST), 'the newest copy, which runs web, is still installed').to.equal(true); + expect(kept?.code, 'the newest copy declined to stand aside').to.equal('NEWEST_HOLDS_WRITER'); + expect(await installedOn(OLDER), 'the next copy is still installed').to.equal(false); + expect(await oneHolderAtMost(), 'what each holder runs').to.deep.equal({ [OLDER]: [], [NEWEST]: ['web'] }); + }); +}); diff --git a/test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js b/test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js new file mode 100644 index 0000000000..b69bed8cd2 --- /dev/null +++ b/test-infra/runner/tests/2250-a-cold-start-seeds-every-g-component-from-the-node-holding-any-of-its-data.js @@ -0,0 +1,96 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { + resetSyncState, getFolderConfig, pinColdStart, pinHolding, +} from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { syncthingSeedIndex } from '../framework/g-app-placement.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A cold start of an app with two single-writer (g:) components, where the only +// data anywhere is one holder's copy of the SECOND component's folder. +// +// A holder ranks the claims on the whole app - every g: folder's holdings summed - +// so the node holding the owner's data seeds every g: component, the empty first +// folder included, and the lowest address seeds none. Ranked folder by folder, the +// empty first folder would fall to the address order and the app would be seeded on +// two nodes, one component each. +// +// The syncthing stub declares each folder's holdings, as suite 2214 does. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const SETTLE_PASSES = 3; +const HOLDING_BYTES = 5821604997; + + +describe('a cold start seeds every g: component from the node holding any of its data', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2egcold${Date.now()}`; + const COMPONENTS = ['db', 'web']; + const HOLDERS = [0, 1]; + const LOWEST = syncthingSeedIndex(HOLDERS); + const HOLDER = HOLDERS.find((i) => i !== LOWEST); + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: COMPONENTS, holders: HOLDERS, + }); + const seeded = (i, name) => client(i).getDecisionCount('syncthing:seedDecision', folderOf(name), 'seeded'); + const folderPasses = (i, name) => client(i).getDecisionCount('syncthing:folderPass', folderOf(name), 'evaluated'); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + + await pushTestApp(appName, 'v1', 'gcold', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: COMPONENTS.map((name) => gComponent(appName, name)), + }); + await Promise.all(COMPONENTS.map((name) => pinColdStart(HOLDERS, folderOf(name)))); + await pinHolding(HOLDER, folderOf('web'), HOLDING_BYTES); + await installOnNodes(env, app, HOLDERS); + }); + + after(async function () { + this.timeout(60000); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + it('seeds and runs every g: component on the holder of the second one\'s data, and nothing on the lowest address', async function () { + this.timeout(600000); + expect(subnet.nodeIp(LOWEST + 1) < subnet.nodeIp(HOLDER + 1), 'fixture: the holder of the data is not the lowest address').to.equal(true); + + await waitHolding(async () => (await oneHolderAtMost())[HOLDER].length === COMPONENTS.length, { + timeout: 300000, interval: 2000, label: 'the holder of the data runs every g: component', + }); + const from = await Promise.all(HOLDERS.flatMap((i) => COMPONENTS.map((name) => folderPasses(i, name)))); + await waitHolding(async () => { + expect(await oneHolderAtMost(), 'what each holder runs').to.deep.equal({ [HOLDER]: COMPONENTS, [LOWEST]: [] }); + const passes = await Promise.all(HOLDERS.flatMap((i) => COMPONENTS.map((name) => folderPasses(i, name)))); + return passes.every((n, k) => n >= from[k] + SETTLE_PASSES); + }, { timeout: 180000, interval: 1000, label: `${SETTLE_PASSES} folder passes over every g: folder on every holder` }); + + for (const name of COMPONENTS) { + // eslint-disable-next-line no-await-in-loop + expect(await seeded(HOLDER, name), `the holder seeded ${name}`).to.be.at.least(1); + // eslint-disable-next-line no-await-in-loop + expect(await seeded(LOWEST, name), `the lowest address seeded ${name}`).to.equal(0); + // eslint-disable-next-line no-await-in-loop + expect((await getFolderConfig(subnet.nodeIp(LOWEST + 1), folderOf(name)))?.type, `the lowest address's ${name} folder`).to.equal('receiveonly'); + } + }); +}); diff --git a/test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js b/test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js new file mode 100644 index 0000000000..9eb87ac8ed --- /dev/null +++ b/test-infra/runner/tests/2251-an-update-applied-in-place-leaves-a-standbys-g-component-stopped.js @@ -0,0 +1,126 @@ +// weight: heavy +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { isDaemonUp, isFolderSynced, getFolderConfig } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A specification update that changes only the plain sibling of an app's single +// single-writer (g:) component, on real syncthing daemons. +// +// Each holder applies it in place: it redeploys the sibling and then restarts the +// app. The restart bounces what this node runs and starts nothing it keeps +// stopped, so the standby's g: component stays down through the update and after +// it, and the primary goes on running it. Started, the app would run on both +// nodes. The plain sibling runs on every holder throughout. +// +// DRIVEN BY BLOCKS: the reinstall pass fires on +// `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0` (suite 85). + +const APP_UID = 1000; + +describe('an update applied in place leaves a standby\'s g: component stopped', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2einplace${Date.now()}`; + const PRIMARY = 0; + const STANDBY = 1; + const HOLDERS = [PRIMARY, STANDBY]; + const web = (content) => ({ + ...gComponent(appName, 'web'), + containerData: '/appdata', + environmentParameters: ['WRITE_FILE=/appdata/web.txt', `WRITE_CONTENT=${content}`], + }); + let app; + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost, holdsFor } = watchGComponents(() => env, { + appName, components: ['db'], holders: HOLDERS, + }); + const plain = watchGComponents(() => env, { appName, components: ['web'], holders: HOLDERS }); + const installedHash = async (i) => { + const res = await client(i).getInstalledApps(); + return (res.data ?? []).find((a) => a.name === appName)?.hash ?? null; + }; + const redeployed = (i, from) => client(i).getEventBuffer().some((e) => e.id > from + && e.event === 'app:componentRedeployed' && e.data?.name === appName && e.data?.component === 'web'); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c, i) => waitFor(() => isDaemonUp(c), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(appName, 'v1', 'inplace', { user: APP_UID }); + app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: [gComponent(appName, 'db'), web('first')], + }); + await installOnNodes(env, app, [PRIMARY]); + await waitHolding(async () => (await oneHolderAtMost())[PRIMARY].includes('db'), { + timeout: 300000, interval: 3000, label: 'the primary runs db', + }); + await installOnNodes(env, app, [STANDBY]); + await waitFor(() => isFolderSynced(client(STANDBY), folderOf('db')), { + timeout: 300000, interval: 3000, label: 'the standby has the primary\'s db data', + }); + await waitFor(async () => (await plain.runningOn(STANDBY)).includes('web'), { + timeout: 180000, interval: 3000, label: 'the standby runs the plain sibling', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('keeps the standby\'s g: component stopped through an update that redeploys its sibling in place', async function () { + // A backstop: the block drive bounds this test (suite 85). + this.timeout(1800000); + const updated = await buildSeedableUpdate(app, (spec) => { + spec.compose[1].environmentParameters = web('second').environmentParameters; + }); + const from = Object.fromEntries(HOLDERS.map((i) => [i, client(i).getLastEventId()])); + await seedSpecUpdate(env, updated, HOLDERS); + await stopTicker(); + + let breach = null; + await driveUntil(client(PRIMARY), async () => { + try { + await oneHolderAtMost(); + const hashes = await Promise.all(HOLDERS.map(installedHash)); + return hashes.every((hash) => hash === updated.hash) && HOLDERS.every((i) => redeployed(i, from[i])); + } catch (error) { + if (error?.name !== 'AssertionError') throw error; + breach = error; + return true; + } + }, { blocks: 216, label: 'both holders redeployed the sibling in place at the new specification' }); + if (breach) throw breach; + + // The restart each holder asked for, carried out. + await holdsFor(60000, { [PRIMARY]: ['db'], [STANDBY]: [] }); + expect((await getFolderConfig(client(STANDBY), folderOf('db')))?.type, 'the standby\'s db folder').to.equal('receiveonly'); + expect((await getFolderConfig(client(PRIMARY), folderOf('db')))?.type, 'the primary\'s db folder').to.equal('sendreceive'); + for (const i of HOLDERS) { + // eslint-disable-next-line no-await-in-loop + expect(await plain.runningOn(i), `node ${i} runs the plain sibling`).to.deep.equal(['web']); + } + }); +}); diff --git a/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js b/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js new file mode 100644 index 0000000000..9e7f27318b --- /dev/null +++ b/test-infra/runner/tests/2252-a-mixed-fleet-seeds-a-cold-start-folder-on-the-holder.js @@ -0,0 +1,134 @@ +// base-image: FLUX_E2E_BASE_TAG +/* + * A fleet part way through an upgrade decides a cold-start seed across both releases. + * + * Three nodes take an r: app at once. The one holding the owner's data runs the released + * FluxOS, which ranks the field on its peers' signed holdings answers but does not read + * which folders a peer has yet to measure. The other two run this build: the one at the + * lowest address, which the address order would make the seed, and a third, held just + * before it measures the folder. + * + * While the third has not measured, the new node waits on it, and the released holder, + * reading the third as answering without a claim, falls to the address order and does + * not lead either: nobody seeds. Once it has measured, both rank the same field and the + * released holder seeds. It reaches that only by verifying the new nodes' answers, + * which carry a field it does not know: an answer it could not verify would read as + * unproven, which blocks its promotion for good. + * + * Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { getAppContainerStatus } from '../framework/container.js'; +import { resetSyncState, pinColdStart, pinHolding } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { syncthingSeedIndex } from '../framework/g-app-placement.js'; +import { buildSeedableSyncthingApp } from '../framework/seed-helper.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; +const subnet = getSubnetConfig(); +const HOLDERS = [0, 1, 2]; +// The holder the address order would pick last, so that seeding it is a decision the +// address order cannot also have produced. It runs the released FluxOS. +const DATA_HOLDER = [...HOLDERS].sort((a, b) => (subnet.nodeIp(a + 1) < subnet.nodeIp(b + 1) ? -1 : 1)).pop(); +const ADDRESS_WINNER = syncthingSeedIndex(HOLDERS); +const UNMEASURED = HOLDERS.find((i) => i !== ADDRESS_WINNER && i !== DATA_HOLDER); +const HOLDER_BYTES = 5821604997; +// The state machine's checkpoint before it measures what this node holds of a +// receive-only folder, keyed by folder id. +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +// Passes over the folder the new address-order winner is counted deciding while the +// third holder is held: more than the leadership confirmation needs, so a win would +// have been acted on. +const HELD_PASSES = 4; + +async function isUp(client, appName) { + const status = await getAppContainerStatus(client.container, appName); + return !!(status && status.status.startsWith('Up')); +} + +function step(what) { + console.log(`# 2252 ${new Date().toISOString()} ${what}`); +} + +describe('a mixed fleet seeds a cold-start folder on the holder', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2emixedseed${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const countUp = async () => (await Promise.all(HOLDERS.map((i) => isUp(env.clients[i], appName)))).filter(Boolean).length; + let parked; + let passesWhileHeld; + let upWhileHeld; + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, nodes: 10, tickerAutostart: false, nodeImageTags: { [DATA_HOLDER]: BASE_TAG }, + }); + if (BASE_TAG === env.nodeImageTag(ADDRESS_WINNER)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG'); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + + await pushImage(appName, 'v1'); + const spec = await buildSeedableSyncthingApp({ name: appName, mode: 'r' }); + await pinColdStart(HOLDERS, folder); + await pinHolding(DATA_HOLDER, folder, HOLDER_BYTES); + + const passes = () => env.clients[ADDRESS_WINNER].getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + await env.clients[UNMEASURED].holdCheckpoint(BEFORE_HOLDINGS, folder); + const from = env.clients[UNMEASURED].getLastEventId(); + try { + step('installing on all three holders, the third held before it measures the folder'); + await installOnNodes(env, spec, HOLDERS); + parked = Boolean(await env.clients[UNMEASURED].waitForEvent( + 'checkpoint:held', (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 120000, { afterId: from }, + ).catch(() => null)); + const passesFrom = await passes(); + await waitFor(async () => (await passes()) >= passesFrom + HELD_PASSES, { + timeout: 120000, interval: 2000, label: `the address-order winner ran ${HELD_PASSES} passes over the folder`, + }).catch(() => {}); + passesWhileHeld = (await passes()) - passesFrom; + upWhileHeld = await countUp(); + } finally { + await env.clients[UNMEASURED].releaseCheckpoint(BEFORE_HOLDINGS, folder); + } + step('released'); + }); + + after(async function () { + this.timeout(60000); + await resetSyncState().catch(() => {}); + await resetFdm().catch(() => {}); + await env?.teardown(); + }); + + it('held the third holder before it measured the folder, while the address-order winner decided it', function () { + expect(parked, 'the third holder was never held, so nothing below was tested').to.equal(true); + expect(passesWhileHeld, 'the address-order winner did not decide the folder while the third was held').to.be.at.least(HELD_PASSES); + }); + + it('seeds nothing while a holder of this build has not measured what it holds', function () { + expect(upWhileHeld, 'a holder seeded while another had not yet measured what it holds').to.equal(0); + }); + + it('seeds the released data holder once every holder has measured, and only it', async function () { + this.timeout(180000); + await waitFor(async () => (await countUp()) >= 1, { + timeout: 120000, interval: 3000, label: 'a holder seeds once every holder has measured what it holds', + }); + await new Promise((r) => { setTimeout(r, 12000); }); + expect(await countUp(), 'more than one holder seeded the folder').to.equal(1); + expect(await isUp(env.clients[DATA_HOLDER], appName), 'the released data holder did not seed').to.equal(true); + }); +}); diff --git a/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js b/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js new file mode 100644 index 0000000000..4997f50275 --- /dev/null +++ b/test-infra/runner/tests/2253-a-holder-whose-volume-did-not-mount-does-not-hold-up-the-seed.js @@ -0,0 +1,115 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { syncthingSeedIndex } from '../framework/g-app-placement.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { destroyFolderVolume } from '../framework/volume-fixture.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A cold start of a g: app where one holder comes back from a restart with the +// app's volume gone: the image is removed, so the mount check cannot repair it and +// holds the folder out of every pass. That holder never measures the folder. +// +// The other holder, the lowest address, decides the seed. It waits on a holder +// that has not yet looked at the folder, so a held-out folder must read as looked +// at with no claim: read as not looked at, it holds the seed up for as long as the +// volume stays gone, and the app starts nowhere. +// +// The lowest address is held before it measures until the broken holder has held +// the folder out. The held pass read its peers before that, so the decision under +// test is the one a later pass makes, held where it opens until the first is done. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const AFTER_FOLDER_READ = 'syncthing:afterFolderRead'; +const SETTLE_PASSES = 6; + +describe('a holder whose volume did not mount does not hold up the seed', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eheldout${Date.now()}`; + const COMPONENT = 'data'; + const HOLDERS = [0, 1]; + const LOWEST = syncthingSeedIndex(HOLDERS); + const BROKEN = HOLDERS.find((i) => i !== LOWEST); + + const client = (i) => env.clients[i]; + const { folderOf, oneHolderAtMost } = watchGComponents(() => env, { + appName, components: [COMPONENT], holders: HOLDERS, + }); + const folder = folderOf(COMPONENT); + const seeded = (i) => client(i).getDecisionCount('syncthing:seedDecision', folder, 'seeded'); + const waitedOnLook = (i) => client(i).getDecisionCount('syncthing:seedDecision', folder, 'holderNotLookedAt'); + const folderPasses = (i) => client(i).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + const heldOutPasses = (i) => client(i).getDecisionCount('syncthing:folderPass', folder, 'mountUnsafe'); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + + await pushTestApp(appName, 'v1', 'heldout', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: HOLDERS.length, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart(HOLDERS, folder); + await client(LOWEST).holdCheckpoint(BEFORE_HOLDINGS, folder); + const from = client(LOWEST).getLastEventId(); + await installOnNodes(env, app, HOLDERS); + await client(LOWEST).waitForEvent( + 'checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, + 300000, + { afterId: from }, + ); + }); + + after(async function () { + this.timeout(60000); + await client(LOWEST)?.releaseAllCheckpoints().catch(() => {}); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + it('seeds on the lowest address while the other holder holds the folder out', async function () { + this.timeout(600000); + expect(subnet.nodeIp(LOWEST + 1) < subnet.nodeIp(BROKEN + 1), 'fixture: the broken holder is not the lowest address').to.equal(true); + + await crashFluxos(client(BROKEN).container, { hold: true }); + await destroyFolderVolume(client(BROKEN).container, folder); + await releaseFluxos(client(BROKEN).container); + await waitFor(async () => await heldOutPasses(BROKEN) >= 1, { + timeout: 300000, interval: 2000, label: 'the restarted holder holds the folder out', + }); + expect(await seeded(LOWEST), 'fixture: the lowest address seeded before it was released').to.equal(0); + + await client(LOWEST).holdCheckpoint(AFTER_FOLDER_READ); + const opened = client(LOWEST).getLastEventId(); + await client(LOWEST).releaseCheckpoint(BEFORE_HOLDINGS, folder); + await client(LOWEST).waitForEvent('checkpoint:held', (d) => d.name === AFTER_FOLDER_READ, 300000, { afterId: opened }); + const waitsBefore = await waitedOnLook(LOWEST); + const from = await folderPasses(LOWEST); + await client(LOWEST).releaseCheckpoint(AFTER_FOLDER_READ); + await waitHolding(async () => { + await oneHolderAtMost(); + return await seeded(LOWEST) >= 1 || await folderPasses(LOWEST) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label: `the lowest address seeds, or ${SETTLE_PASSES} folder passes go by without it` }); + + expect(await seeded(LOWEST), `seeds on the lowest address in ${SETTLE_PASSES} passes (passes it waited on a holder that had not looked: ${await waitedOnLook(LOWEST) - waitsBefore})`).to.be.at.least(1); + expect(await waitedOnLook(LOWEST) - waitsBefore, 'passes the lowest address waited on the held-out holder').to.equal(0); + expect(await seeded(BROKEN), 'the broken holder seeded').to.equal(0); + expect(await heldOutPasses(BROKEN), 'fixture: the broken holder still holds the folder out').to.be.at.least(1); + }); +}); diff --git a/test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js b/test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js new file mode 100644 index 0000000000..a93377f29e --- /dev/null +++ b/test-infra/runner/tests/2254-a-holder-that-cannot-mount-its-volume-is-left-out-of-the-seed.js @@ -0,0 +1,225 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart, pinHolding } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { breakFolderVolume, destroyFolderVolume, restoreFolderVolume } from '../framework/volume-fixture.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Cold starts of a g: app where a holder comes back from a restart with a volume it +// cannot mount, and its address sorts first, so the address order would give it the +// seed. It cannot seed. +// +// A holder whose volume image is still on it may hold the app's only copy: it is left +// out of the choice, a holder with data seeds, and while no other holder has data +// nobody seeds until the volume mounts. A holder whose image is gone holds nothing: +// it is left out, and the rest seed as they would without it. +// +// Every other holder is held before it measures until each broken holder has held its +// folder out, so the decisions under test are made with the broken holders' answers. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const COMPONENT = 'data'; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; +const DATA_BYTES = 5821604997; +// A broken holder whose image is on it but will not mount, and one whose image is gone. +const UNMOUNTABLE = 'unmountable'; +const GONE = 'gone'; + +describe('a holder that cannot mount its volume is left out of the seed', function () { + let env; + dumpLogsOnFailure(() => env); + const client = (i) => env.clients[i]; + const byAddress = (holders) => [...holders].sort((a, b) => (subnet.nodeIp(a + 1) < subnet.nodeIp(b + 1) ? -1 : 1)); + + // One app per case, on its own holders, decided from a cold start. + function coldStartCase(tag, { holders, broken, dataOn }) { + const appName = `e2ecannotseed${tag}${Date.now()}`; + const watch = watchGComponents(() => env, { appName, components: [COMPONENT], holders }); + const folder = watch.folderOf(COMPONENT); + const count = (i, counter, decision) => client(i).getDecisionCount(counter, folder, decision); + const brokenHolders = Object.keys(broken).map(Number); + const deciders = holders.filter((i) => !brokenHolders.includes(i)); + const seeded = (i) => count(i, 'syncthing:seedDecision', 'seeded'); + + return { + folder, + oneHolderAtMost: watch.oneHolderAtMost, + seeded, + cannotSeed: (i) => count(i, 'syncthing:seedDecision', 'holderCannotSeed'), + waitedOnVolume: (i) => count(i, 'syncthing:seedDecision', 'holderVolumeHeldOut'), + passes: (i) => count(i, 'syncthing:folderPass', 'evaluated'), + heldOutPasses: async (i) => await count(i, 'syncthing:folderPass', 'mountUnsafe') + await count(i, 'syncthing:folderPass', 'volumeNotMounted'), + seeders: async () => (await Promise.all(holders.map(seeded))).map((n, k) => (n > 0 ? holders[k] : null)).filter((i) => i !== null), + + async start() { + expect(byAddress(holders)[0], 'fixture: a broken holder is not the lowest address').to.be.oneOf(brokenHolders); + await pushTestApp(appName, 'v1', 'cannotseed', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: holders.length, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart(holders, folder); + if (dataOn !== undefined) await pinHolding(dataOn, folder, DATA_BYTES); + const from = Object.fromEntries(deciders.map((i) => [i, client(i).getLastEventId()])); + await Promise.all(deciders.map((i) => client(i).holdCheckpoint(BEFORE_HOLDINGS, folder))); + await installOnNodes(env, app, holders); + await Promise.all(deciders.map((i) => client(i).waitForEvent( + 'checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, + 300000, + { afterId: from[i] }, + ))); + + // eslint-disable-next-line no-restricted-syntax + for (const i of brokenHolders) { + // eslint-disable-next-line no-await-in-loop + await crashFluxos(client(i).container, { hold: true }); + // eslint-disable-next-line no-await-in-loop + if (broken[i] === GONE) await destroyFolderVolume(client(i).container, folder); + // eslint-disable-next-line no-await-in-loop + else await breakFolderVolume(client(i).container, folder); + // eslint-disable-next-line no-await-in-loop + await releaseFluxos(client(i).container); + // eslint-disable-next-line no-await-in-loop + await waitFor(async () => await count(i, 'syncthing:folderPass', 'mountUnsafe') >= 1, { + timeout: 300000, interval: 2000, label: `node ${i} holds the folder out`, + }); + } + await Promise.all(deciders.map((i) => client(i).releaseCheckpoint(BEFORE_HOLDINGS, folder))); + }, + + // Until `done`, or SETTLE_PASSES folder passes go by on `watched`; never more than + // one holder running the app. + async settle(watched, done, label) { + const from = await this.passes(watched); + await waitHolding(async () => { + await watch.oneHolderAtMost(); + return await done() || await this.passes(watched) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label }); + }, + }; + } + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + }); + + after(async function () { + this.timeout(60000); + await Promise.all((env?.clients ?? []).map((c) => c?.releaseAllCheckpoints().catch(() => {}))); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + describe('the lowest address cannot mount a volume whose image is on it, and another holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('data', { holders: [0, 1], broken: { 0: UNMOUNTABLE }, dataOn: 1 }))); + + it('seeds on the holder with data', async function () { + this.timeout(900000); + await c.start(); + await c.settle(1, async () => await c.seeded(1) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeded(1), `the holder with data did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.cannotSeed(1), 'the holder with data never left the broken holder out').to.be.at.least(1); + expect(await c.seeded(0), 'the broken holder seeded').to.equal(0); + expect(await c.heldOutPasses(0), 'fixture: the broken holder no longer holds the folder out').to.be.at.least(1); + }); + + it('takes the broken holder back as a standby once its volume mounts', async function () { + this.timeout(600000); + const from = await c.passes(0); + await restoreFolderVolume(client(0).container, c.folder); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return await c.passes(0) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label: `the restored holder passes its folder ${SETTLE_PASSES} times` }); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([1]); + }); + }); + + describe('the lowest address cannot mount a volume whose image is on it, and no other holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('wait', { holders: [2, 3], broken: { 2: UNMOUNTABLE } }))); + + it('seeds nothing while the volume will not mount', async function () { + this.timeout(900000); + await c.start(); + await c.settle(3, async () => await c.seeded(3) >= 1, 'the other holder seeds, or the passes go by without it'); + + expect(await c.seeded(3), 'an empty copy seeded over a volume that may hold the app\'s data').to.equal(0); + expect(await c.waitedOnVolume(3), 'the other holder never waited on the broken holder\'s volume').to.be.at.least(1); + expect(await c.seeded(2), 'the broken holder seeded').to.equal(0); + }); + + it('seeds once that volume mounts', async function () { + this.timeout(600000); + await restoreFolderVolume(client(2).container, c.folder); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return (await c.seeders()).length >= 1; + }, { timeout: 300000, interval: 2000, label: 'a holder seeds once the volume mounts' }); + + expect(await c.seeders(), 'the holders that seeded').to.have.lengthOf(1); + }); + }); + + describe('the lowest address\'s volume image is gone, and no other holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('gone', { holders: [4, 5], broken: { 4: GONE } }))); + + it('seeds on the other holder, empty', async function () { + this.timeout(900000); + await c.start(); + await c.settle(5, async () => await c.seeded(5) >= 1, 'the other holder seeds, or the passes go by without it'); + + expect(await c.seeded(5), `the other holder did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.cannotSeed(5), 'the other holder never left the broken holder out').to.be.at.least(1); + expect(await c.waitedOnVolume(5), 'the other holder waited on a volume whose image is gone').to.equal(0); + expect(await c.seeded(4), 'the broken holder seeded').to.equal(0); + }); + }); + + describe('the lowest address cannot mount, the next holds nothing, and the last has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('three', { holders: [1, 3, 5], broken: { 1: UNMOUNTABLE }, dataOn: 5 }))); + + it('seeds on the holder with data, not the next address', async function () { + this.timeout(900000); + await c.start(); + await c.settle(5, async () => await c.seeded(5) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([5]); + }); + }); + + describe('two holders cannot mount, one with its image and one without, and the last has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('two', { holders: [0, 2, 4], broken: { 0: UNMOUNTABLE, 2: GONE }, dataOn: 4 }))); + + it('seeds on the holder with data', async function () { + this.timeout(900000); + await c.start(); + await c.settle(4, async () => await c.seeded(4) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([4]); + }); + }); +}); diff --git a/test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js b/test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js new file mode 100644 index 0000000000..fb3e1866be --- /dev/null +++ b/test-infra/runner/tests/2255-a-holder-that-cannot-read-its-locations-keeps-what-it-tells-its-peers.js @@ -0,0 +1,145 @@ +// weight: heavy +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Cold starts of a g: app where the holder at the higher address cannot read its +// locations: every read of its location records fails. Its monitor holds the folder +// out of each pass, and what it tells its peers of the folder stays as it was - a +// read that failed says nothing about what the volume holds. +// +// The lowest address decides the seed. Beside a holder that measured the folder +// before its reads failed, it seeds. Beside one that has not looked at the folder +// since its FluxOS restarted, it waits, and seeds once that holder can read again. +// +// The lowest address is held before it measures until the other holder's reads have +// failed, so the decisions under test are made with the answer that holder gives then. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const COMPONENT = 'data'; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; +const unreadableLocations = (i) => failCommands(i + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); + +describe('a holder that cannot read its locations keeps what it tells its peers', function () { + let env; + let release = null; + dumpLogsOnFailure(() => env); + const client = (i) => env.clients[i]; + + // One app per case, on its own two holders: `lowest` decides, `other` cannot read. + function coldStartCase(tag, { lowest, other }) { + const appName = `e2elocunread${tag}${Date.now()}`; + const watch = watchGComponents(() => env, { appName, components: [COMPONENT], holders: [lowest, other] }); + const folder = watch.folderOf(COMPONENT); + const count = (i, counter, decision) => client(i).getDecisionCount(counter, folder, decision); + return { + folder, + seeded: (i) => count(i, 'syncthing:seedDecision', 'seeded'), + waitedOnLook: (i) => count(i, 'syncthing:seedDecision', 'holderNotLookedAt'), + waitedOnVolume: (i) => count(i, 'syncthing:seedDecision', 'holderVolumeHeldOut'), + passes: (i) => count(i, 'syncthing:folderPass', 'evaluated'), + unread: (i) => count(i, 'syncthing:folderPass', 'locationsUnread'), + + // Placed on both, the lowest held before it measures. + async place() { + expect(subnet.nodeIp(lowest + 1) < subnet.nodeIp(other + 1), 'fixture: the deciding holder is not the lowest address').to.equal(true); + await pushTestApp(appName, 'v1', 'locunread', { user: APP_UID }); + const app = await buildSeedableApp({ + env, name: appName, instances: 2, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart([lowest, other], folder); + const from = client(lowest).getLastEventId(); + await client(lowest).holdCheckpoint(BEFORE_HOLDINGS, folder); + await installOnNodes(env, app, [lowest, other]); + await client(lowest).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 300000, { afterId: from }); + await waitFor(async () => await this.passes(other) >= 2, { + timeout: 300000, interval: 2000, label: 'the other holder has measured the folder', + }); + }, + + // The lowest released, until `done` or SETTLE_PASSES of its passes go by. + async decide(done, label) { + const from = await this.passes(lowest); + await client(lowest).releaseCheckpoint(BEFORE_HOLDINGS, folder); + await waitHolding(async () => { + await watch.oneHolderAtMost(); + return await done() || await this.passes(lowest) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label }); + }, + }; + } + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 6, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + }); + + afterEach(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + release = null; + await Promise.all((env?.clients ?? []).map((c) => c?.releaseAllCheckpoints().catch(() => {}))); + }); + + after(async function () { + this.timeout(60000); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + it('seeds beside a holder that measured the folder before its reads failed', async function () { + this.timeout(900000); + const c = coldStartCase('measured', { lowest: 1, other: 2 }); + await c.place(); + release = await unreadableLocations(2); + await waitFor(async () => await c.unread(2) >= 1, { + timeout: 120000, interval: 2000, label: 'the other holder cannot read its locations', + }); + + await c.decide(async () => await c.seeded(1) >= 1, 'the lowest address seeds, or the passes go by without it'); + + expect(await c.seeded(1), `the lowest address did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.waitedOnVolume(1), 'the lowest address waited on a holder that only could not read its locations').to.equal(0); + }); + + it('waits on a holder that has not looked at the folder since it restarted, and seeds once it can read again', async function () { + this.timeout(900000); + const c = coldStartCase('notlooked', { lowest: 3, other: 4 }); + await c.place(); + await crashFluxos(client(4).container, { hold: true }); + release = await unreadableLocations(4); + await releaseFluxos(client(4).container); + await waitFor(async () => await c.unread(4) >= 1, { + timeout: 300000, interval: 2000, label: 'the restarted holder cannot read its locations', + }); + + const waitsBefore = await c.waitedOnLook(3); + await c.decide(async () => await c.seeded(3) >= 1, 'the lowest address seeds, or the passes go by without it'); + expect(await c.seeded(3), 'seeded beside a holder that has not looked at the folder').to.equal(0); + expect(await c.waitedOnLook(3) - waitsBefore, 'the lowest address never waited on the holder that has not looked').to.be.at.least(1); + + await release(); + release = null; + await waitFor(async () => await c.seeded(3) >= 1, { + timeout: 300000, interval: 2000, label: 'the lowest address seeds once the other holder can read its locations', + }); + }); +}); diff --git a/test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js b/test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js new file mode 100644 index 0000000000..1756dea0fa --- /dev/null +++ b/test-infra/runner/tests/2256-a-holder-that-cannot-decrypt-an-apps-spec-is-left-out-of-the-seed.js @@ -0,0 +1,174 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { resetSyncState, pinColdStart, pinHolding } from '../framework/syncthing-control.js'; +import { resetFdm } from '../framework/fdm-control.js'; +import { failRpc, clearRpcFailures } from '../framework/daemon-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitHolding } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { buildSeedableEnterpriseApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { crashFluxos, releaseFluxos } from '../framework/container.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Cold starts of an enterprise g: app where a holder comes back from a restart unable +// to decrypt the app's spec: the benchmark refuses it the key. Its monitor pass does +// not reach the app's folder, so what it holds is unknown. +// +// It is left out of the choice: a holder with data seeds, and while no other holder +// has data nobody seeds until it can decrypt again. Once it can, it measures the +// folder and the seed goes ahead. +// +// Every other holder is held before it measures until each broken holder answers its +// peers again, so the decisions under test are made with the broken holders' answers. + +const subnet = getSubnetConfig(); +const APP_UID = 1000; +const COMPONENT = 'data'; +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +const SETTLE_PASSES = 6; +const DATA_BYTES = 5821604997; +const DECRYPT = 'decryptrsamessage'; +const nodeIp = (i) => subnet.nodeIp(i + 1); + +describe('a holder that cannot decrypt an app\'s spec is left out of the seed', function () { + let env; + dumpLogsOnFailure(() => env); + const client = (i) => env.clients[i]; + + // One app per case, on its own two holders, decided from a cold start. + function coldStartCase(tag, { lowest, other, broken, dataOn }) { + const holders = [lowest, other]; + const appName = `e2enodecrypt${tag}${Date.now()}`; + const watch = watchGComponents(() => env, { appName, components: [COMPONENT], holders }); + const folder = watch.folderOf(COMPONENT); + const count = (i, counter, decision) => client(i).getDecisionCount(counter, folder, decision); + const decider = holders.find((i) => i !== broken); + const seeded = (i) => count(i, 'syncthing:seedDecision', 'seeded'); + + return { + folder, + decider, + oneHolderAtMost: watch.oneHolderAtMost, + seeded, + cannotSeed: (i) => count(i, 'syncthing:seedDecision', 'holderCannotSeed'), + waitedOnVolume: (i) => count(i, 'syncthing:seedDecision', 'holderVolumeHeldOut'), + passes: (i) => count(i, 'syncthing:folderPass', 'evaluated'), + seeders: async () => (await Promise.all(holders.map(seeded))).map((n, k) => (n > 0 ? holders[k] : null)).filter((i) => i !== null), + + async start() { + expect(nodeIp(lowest) < nodeIp(other), 'fixture: the lowest holder is not the lowest address').to.equal(true); + await pushTestApp(appName, 'v1', 'nodecrypt', { user: APP_UID }); + const app = await buildSeedableEnterpriseApp({ + env, name: appName, instances: holders.length, compose: [gComponent(appName, COMPONENT)], + }); + await pinColdStart(holders, folder); + if (dataOn !== undefined) await pinHolding(dataOn, folder, DATA_BYTES); + const from = client(decider).getLastEventId(); + await client(decider).holdCheckpoint(BEFORE_HOLDINGS, folder); + await installOnNodes(env, app, holders); + await client(decider).waitForEvent( + 'checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, + 300000, + { afterId: from }, + ); + + await crashFluxos(client(broken).container, { hold: true }); + await failRpc(nodeIp(broken), DECRYPT, { times: 1000000 }); + await releaseFluxos(client(broken).container); + await waitFor(async () => (await client(broken).getPromotedFolders())?.data?.ready === true, { + timeout: 300000, interval: 2000, label: `node ${broken} answers its peers after its restart`, + }); + await client(decider).releaseCheckpoint(BEFORE_HOLDINGS, folder); + }, + + // Until `done`, or SETTLE_PASSES folder passes go by on the decider; never more + // than one holder running the app. + async settle(done, label) { + const from = await this.passes(decider); + await waitHolding(async () => { + await watch.oneHolderAtMost(); + return await done() || await this.passes(decider) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label }); + }, + }; + } + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 4, tickerAutostart: false }); + await bootAndPeer(env); + await resetFdm(); + await resetSyncState(); + }); + + after(async function () { + this.timeout(60000); + await Promise.all((env?.clients ?? []).map((c) => c?.releaseAllCheckpoints().catch(() => {}))); + await Promise.all([0, 1, 2, 3].map((i) => clearRpcFailures(nodeIp(i)).catch(() => {}))); + await resetSyncState().catch(() => {}); + await env?.teardown(); + }); + + describe('the lowest address cannot decrypt the spec, and another holder has data', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('data', { + lowest: 0, other: 1, broken: 0, dataOn: 1, + }))); + + it('seeds on the holder with data', async function () { + this.timeout(900000); + await c.start(); + await c.settle(async () => await c.seeded(1) >= 1, 'the holder with data seeds, or the passes go by without it'); + + expect(await c.seeded(1), `the holder with data did not seed in ${SETTLE_PASSES} passes`).to.be.at.least(1); + expect(await c.seeded(0), 'the holder that cannot decrypt seeded').to.equal(0); + }); + + it('takes the holder back as a standby once it can decrypt', async function () { + this.timeout(600000); + const from = await c.passes(0); + await clearRpcFailures(nodeIp(0)); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return await c.passes(0) >= from + SETTLE_PASSES; + }, { timeout: 300000, interval: 2000, label: `the holder that can decrypt again passes its folder ${SETTLE_PASSES} times` }); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([1]); + }); + }); + + describe('a holder that cannot decrypt the spec may hold the data, and the lowest address has none', () => { + const c = {}; + before(() => Object.assign(c, coldStartCase('wait', { + lowest: 2, other: 3, broken: 3, dataOn: 3, + }))); + + it('seeds nothing while that holder cannot decrypt', async function () { + this.timeout(900000); + await c.start(); + await c.settle(async () => await c.seeded(2) >= 1, 'the lowest address seeds, or the passes go by without it'); + + expect(await c.seeded(2), 'an empty copy seeded over a folder that may hold the app\'s data').to.equal(0); + expect(await c.waitedOnVolume(2), 'the lowest address never waited on the holder that cannot decrypt').to.be.at.least(1); + expect(await c.seeded(3), 'the holder that cannot decrypt seeded').to.equal(0); + }); + + it('seeds on the holder with data once it can decrypt', async function () { + this.timeout(600000); + await clearRpcFailures(nodeIp(3)); + await waitHolding(async () => { + await c.oneHolderAtMost(); + return (await c.seeders()).length >= 1; + }, { timeout: 300000, interval: 2000, label: 'a holder seeds once the spec can be decrypted' }); + + expect(await c.seeders(), 'the holders that seeded').to.deep.equal([3]); + }); + }); +}); diff --git a/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js b/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js index ad92278b97..e73f947dea 100644 --- a/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js +++ b/test-infra/runner/tests/2302-a-standby-whose-calls-reach-another-node.js @@ -53,7 +53,7 @@ const appDir = (name) => `/mnt/appdata/flux-apps/flux${name}_${name}`; // Where a node's syncthing monitor took each device it looked up for `name`, // counted per source. -const DEVICE_SOURCES = ['verified', 'verifiedNoDevice', 'introduced', 'withheld', 'held', 'unsigned']; +const DEVICE_SOURCES = ['verified', 'verifiedNoDevice', 'introduced', 'withheld', 'held', 'unsigned', 'configured']; async function deviceSources(client, name) { const counts = await Promise.all(DEVICE_SOURCES.map((source) => client.getDecisionCount('syncthing:deviceSource', name, source))); return Object.fromEntries(DEVICE_SOURCES.map((source, i) => [source, counts[i]])); diff --git a/test-infra/runner/tests/2401-legacy-maintenance-access.js b/test-infra/runner/tests/2401-legacy-maintenance-access.js index 1a33784e8a..8017438049 100644 --- a/test-infra/runner/tests/2401-legacy-maintenance-access.js +++ b/test-infra/runner/tests/2401-legacy-maintenance-access.js @@ -1,4 +1,4 @@ -// Maintenance SSH access on a legacy node, end to end on systemd-mode nodes: the +// Maintenance SSH access on a legacy node, end to end: the // fluxadm account, its passwordless sudo, the configured keys and a dedicated sshd // unit on apiport - 5, reached over the fleet network with a real ssh client. // @@ -327,7 +327,6 @@ describe('2401 legacy node maintenance access', function suite() { nodes: 7, legacyNodes: [LEGACY, NODE_OWNER_SSHD, ...FAULTED], firewall: [LEGACY], - systemdMode: true, tickerAutostart: false, configOverrides: { fluxadm: { sshAuthorizedKeys: [publicKey('current')] } }, }); diff --git a/test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js b/test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js new file mode 100644 index 0000000000..34dbe1aba4 --- /dev/null +++ b/test-infra/runner/tests/2507-a-node-without-dockers-systemd-cgroup-driver-holds-itself-out-of-service.js @@ -0,0 +1,65 @@ +// A node whose cgroups are not the ones FluxOS runs apps in holds itself out of +// service and says why. Harness nodes run cgroup v2, with cgroup.kill, under Docker's +// systemd cgroup driver, so the node under test has its Docker switched to the +// cgroupfs driver and FluxOS restarted on it; the node beside it keeps the systemd +// driver: the canary that the check passes a node that meets it. The fleet boots +// first, since a node out of service refuses the login the harness boots a fleet +// through. +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, restartDockerd, restartFluxos } from '../framework/container.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const OFF_DRIVER = 0; +const ON_DRIVER = 1; + +const cgroupDriver = async (client) => (await execInContainer(client.container, 'docker info -f {{.CgroupDriver}}')).stdout.trim(); + +describe('2507 a node without Docker\'s systemd cgroup driver holds itself out of service', function suite() { + this.timeout(600000); + + let env; + dumpLogsOnFailure(() => env); + + before(async function hook() { + env = await createTestEnv({ hookCtx: this, nodes: 2, tickerAutostart: false }); + await Promise.all(env.clients.map((client) => waitForBootSettled(client, 240000))); + const node = env.clients[OFF_DRIVER]; + // The driver set in the node's own daemon.json, merged over what the entrypoint wrote. + const switched = await execInContainer(node.container, `node -e ' + const fs = require("fs"); + const target = "/etc/docker/daemon.json"; + const config = fs.existsSync(target) ? JSON.parse(fs.readFileSync(target, "utf8")) : {}; + config["exec-opts"] = ["native.cgroupdriver=cgroupfs"]; + fs.writeFileSync(target, JSON.stringify(config, null, 2)); + '`); + expect(switched.exitCode, switched.stderr).to.equal(0); + await restartDockerd(node.container); + expect(await cgroupDriver(node), 'fixture: Docker did not take the cgroupfs driver').to.equal('cgroupfs'); + await restartFluxos(node.container); + }); + + after(async () => { + await env?.teardown(); + }); + + it('holds the node at DOS 100, naming the driver it found, the one required, and what to do', async () => { + let state; + await waitFor(async () => { + state = (await env.clients[OFF_DRIVER].getDOSState())?.data; + return state?.dosState === 100; + }, { timeout: 90000, label: 'the node without the systemd driver to hold itself out of service' }); + expect(state.dosMessage).to.include('Cgroup Error'); + expect(state.dosMessage).to.include('Docker\'s systemd cgroup driver (found cgroupfs)'); + expect(state.dosMessage).to.include('restart FluxOS'); + }); + + it('does not hold the node whose Docker uses the systemd driver', async () => { + expect(await cgroupDriver(env.clients[ON_DRIVER]), 'fixture: the canary\'s Docker').to.equal('systemd'); + const state = (await env.clients[ON_DRIVER].getDOSState()).data; + expect(state.dosState, state.dosMessage).to.not.equal(100); + expect(String(state.dosMessage ?? '')).to.not.include('Cgroup Error'); + }); +}); diff --git a/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js new file mode 100644 index 0000000000..394909d586 --- /dev/null +++ b/test-infra/runner/tests/3001-a-node-behind-a-full-router-table-verifies-upnp.js @@ -0,0 +1,168 @@ +// weight: light +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const subnet = getSubnetConfig(); + +// A node on any api port but 16127 runs only if it verifies UPnP, and it verifies by mapping +// a test port. Its router's table is full: the test mappings earlier runs of FluxOS left on +// it, and one mapping of another machine on the network. Until UPnP is verified the stale +// mapping cleanup never runs, so the verification itself makes room: it removes this node's +// left-over test mappings, maps its test port, and the node maps its own ports in the room +// that leaves. The other machine's mapping is not this node's to remove. + +const API_PORT = 16137; +// The node maps no other address: upnp.nodeIp is empty in the harness, so its mappings +// point at the address the client is given in its place. +const NODE_CLIENT = '127.0.0.1'; +const OTHER_MACHINE = subnet.nodeIp(9); + +const mapping = (externalPort, description, internalClient = NODE_CLIENT) => ({ + externalPort, protocol: 'TCP', internalPort: externalPort, internalClient, description, leaseDuration: 0, +}); +const LEFT_OVER = [ + mapping(31000, 'Flux_Test_App'), + mapping(31001, 'Flux_Prelaunch_App_31001'), + mapping(31002, 'Flux_Test_App'), + mapping(31003, 'Flux_UPNP_Mapping_Test'), +]; +const OTHERS = mapping(32000, 'Flux_App_sibling', OTHER_MACHINE); +// Full as it starts, and room for the node's four ports once the left-overs are gone. +const CAPACITY = LEFT_OVER.length + 1; +if (CAPACITY - 1 < 4) throw new Error('3001: the table must have room for the four core ports once cleared'); + +describe('a node behind a full router table verifies UPnP', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); + await enableUpnpGateway(nodeIp, { capacity: CAPACITY, mappings: [...LEFT_OVER, OTHERS] }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('removes its left-over test mappings and maps its own ports in their room', async function () { + this.timeout(180000); + const corePorts = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2]; + let table = []; + await waitFor(async () => { + table = (await upnpGatewayState(nodeIp)).mappings; + return corePorts.every((port) => table.some((m) => m.externalPort === port)); + }, { timeout: 150000, interval: 2000, label: 'the node\'s own ports mapped' }).catch(() => {}); + + expect(table.map((m) => m.externalPort).sort(), 'the router\'s table').to.deep.equal([...corePorts, OTHERS.externalPort].sort()); + }); + + it('leaves the other machine\'s mapping as it was', async function () { + const { mappings, requests } = await upnpGatewayState(nodeIp); + + expect(requests.some((r) => r.action === 'GetGenericPortMappingEntry'), 'canary: the node listed the table').to.equal(true); + expect(mappings.find((m) => m.externalPort === OTHERS.externalPort)).to.deep.equal(OTHERS); + }); +}); + +// The same full table on a router that stores descriptions as a FRITZ!Box does, its +// underscores and spaces as dots: the left-over test mappings read 'Flux.Test.App' and the +// like, and are this node's all the same. +describe('a node behind a full FRITZ!Box-style router table verifies UPnP', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + const fritz = (m) => ({ ...m, description: m.description.replace(/_/g, '.') }); + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); + await enableUpnpGateway(nodeIp, { + capacity: CAPACITY, mappings: [...LEFT_OVER.map(fritz), fritz(OTHERS)], fritzDescriptions: true, + }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('removes its left-over test mappings and maps its own ports in their room', async function () { + this.timeout(180000); + const corePorts = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2]; + let table = []; + await waitFor(async () => { + table = (await upnpGatewayState(nodeIp)).mappings; + return corePorts.every((port) => table.some((m) => m.externalPort === port)); + }, { timeout: 150000, interval: 2000, label: 'the node\'s own ports mapped' }).catch(() => {}); + + expect(table.map((m) => m.externalPort).sort(), 'the router\'s table').to.deep.equal([...corePorts, OTHERS.externalPort].sort()); + expect(table.find((m) => m.externalPort === API_PORT).description, 'canary: the router stores dots').to.equal('Flux.Backend.API'); + }); +}); + +// The same router with nothing of this node's to remove: it holds the node's own four ports, +// mapped by an earlier run, and other machines' mappings up to its capacity. The test mapping +// finds no room, and the router already maps this node's api port to it, so the node runs on +// what it holds. +describe('a node whose own ports fill its router\'s table verifies UPnP', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + const OWN = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2] + .map((port, i) => mapping(port, ['Flux_Home_UI', 'Flux_Backend_API', 'Flux_Backend_API_SSL', 'Flux_Syncthing'][i])); + const OTHER_MACHINES = [32000, 32001].map((port) => mapping(port, 'Flux_App_sibling', OTHER_MACHINE)); + const FULL = [...OWN, ...OTHER_MACHINES]; + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); + await enableUpnpGateway(nodeIp, { capacity: FULL.length, mappings: FULL }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('starts on its api port, its mappings and the other machines\' as they were', async function () { + this.timeout(180000); + // past the 120s a node that failed verification waits before it exits + await new Promise((resolve) => { setTimeout(resolve, 150000); }); + const { mappings, requests } = await upnpGatewayState(nodeIp); + + expect(requests.some((r) => r.action === 'AddPortMapping' && r.port === API_PORT + 3), 'canary: the node asked for its test mapping').to.equal(true); + // a node that failed verification never maps its own ports + expect(requests.some((r) => r.action === 'AddPortMapping' && r.port === API_PORT), 'the node mapped its api port').to.equal(true); + expect(mappings.map((m) => m.externalPort).sort()).to.deep.equal(FULL.map((m) => m.externalPort).sort()); + expect(requests.filter((r) => r.action === 'DeletePortMapping'), 'nothing removed').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js b/test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js new file mode 100644 index 0000000000..94f55430f4 --- /dev/null +++ b/test-infra/runner/tests/3002-a-node-asks-again-for-a-removal-its-router-refused.js @@ -0,0 +1,76 @@ +// weight: light +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { enableUpnpGateway, upnpGatewayState } from '../framework/external-http-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { waitFor, waitForBootSettled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const subnet = getSubnetConfig(); + +// A node verifies UPnP by mapping a test port, reading it back and removing it. Its router +// refuses the first removals it is asked for. The node verifies all the same - the router +// took the mapping and showed it - and asks again for the removal, waiting longer each +// time (test-infra/config/shared.js: 2s, doubling to 8s), until the router removes it. + +const API_PORT = 16137; +const TEST_PORT = API_PORT + 3; +const REFUSED = 2; +const FIRST_RETRY_MS = 2000; + +describe('a node asks again for a removal its router refused', function () { + let env; + dumpLogsOnFailure(() => env); + let nodeIp; + + before(async function () { + this.timeout(240000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + nodeIp = subnet.nodeIp(1); + // the first boot writes userconfig.js (its identity) and would put back the old port + await waitForBootSettled(env.clients[0]); + await enableUpnpGateway(nodeIp, { refuseDeletes: REFUSED }); + await setNodeApiPort(env.clients[0].container, API_PORT); + await restartFluxos(env.clients[0].container, { apiPort: API_PORT }); + await env.clients[0].followTo(nodeIp, API_PORT); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('verifies, and removes its test mapping once the router takes the removal', async function () { + this.timeout(180000); + const corePorts = [API_PORT - 1, API_PORT, API_PORT + 1, API_PORT + 2]; + let state = { mappings: [], requests: [] }; + const removalsOfTestPort = () => state.requests.filter((r) => r.action === 'DeletePortMapping' && r.port === TEST_PORT); + await waitFor(async () => { + state = await upnpGatewayState(nodeIp); + return removalsOfTestPort().length > REFUSED + && corePorts.every((port) => state.mappings.some((m) => m.externalPort === port)); + }, { timeout: 150000, interval: 1000, label: 'the test mapping removed and the core ports mapped' }).catch(() => {}); + + const asks = removalsOfTestPort(); + expect(asks.filter((r) => r.refused === 501).length, 'canary: the router refused the first removals').to.equal(REFUSED); + expect(asks.length, 'asked again until the router took it').to.equal(REFUSED + 1); + // a node that failed verification never maps its own ports + expect(corePorts.every((port) => state.mappings.some((m) => m.externalPort === port)), 'verified: its own ports mapped').to.equal(true); + expect(state.mappings.some((m) => m.externalPort === TEST_PORT), 'the test mapping is gone').to.equal(false); + + const gaps = asks.slice(1).map((r, i) => Date.parse(r.at) - Date.parse(asks[i].at)); + expect(gaps[0], 'the first ask again waits the first retry').to.be.at.least(FIRST_RETRY_MS - 250); + expect(gaps[1], 'the next waits twice as long').to.be.at.least(2 * FIRST_RETRY_MS - 250); + }); + + it('asks for no other removal of the test port', async function () { + this.timeout(30000); + await new Promise((resolve) => { setTimeout(resolve, 10000); }); + const { requests } = await upnpGatewayState(nodeIp); + expect(requests.filter((r) => r.action === 'DeletePortMapping' && r.port === TEST_PORT).length).to.equal(REFUSED + 1); + }); +}); diff --git a/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js new file mode 100644 index 0000000000..ab8f997a66 --- /dev/null +++ b/test-infra/runner/tests/3003-a-node-whose-router-refuses-it-is-out-of-service-until-it-stops.js @@ -0,0 +1,294 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { pushBusybox } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { + enableUpnpGateway, setUpnpCapacity, setUpnpDeleteFault, setUpnpLookupFault, setUpnpRefuseDeletes, + upnpGatewayState, +} from '../framework/external-http-control.js'; +import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; +import { + waitFor, waitForAppInstalled, waitForAppRemoved, waitForBootSettled, +} from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; +import { authenticate } from '../auth.js'; +import { fluxTeamKey } from '../framework/keys.js'; + +const subnet = getSubnetConfig(); + +// A node behind UPnP whose router refuses it - to remove a mapping, or to make +// one - is out of service until the router stops. Its availability check waits +// ten minutes after a passing test and five in DOS, so only UPnP reaching it +// can start a pass sooner than that. +// +// An app's removal the router refuses starts the next pass at once; each pass +// while it stays refused makes no test mapping and counts toward the DOS, and +// the router taking the removal starts the pass that tests and releases the +// node. An app's mapping a full router refuses starts the next pass at once; +// that pass's own test mapping is refused too, and so is every one after it +// until the router has room, when the next pass releases the node. A router +// that carries out a removal but answers it with a fault - and answers the +// same for an entry it no longer holds - leaves nothing pending: the retry +// finds the entry gone, and the node stays in service. A lookup the router +// answers with a fault settles nothing: the mapping still counts as the +// node's, the node is out of service while the router refuses to remove it, +// and back once the router has. + +const NODE = 0; +const PASSED_WAIT_MS = 10 * 60 * 1000; +const DOS_WAIT_MS = 5 * 60 * 1000; +const APPS_PORT_CHECK = { + timeoutsMs: { + default: PASSED_WAIT_MS, error: 5000, failure: 1000, dos: DOS_WAIT_MS, appError: 5000, + }, + thresholds: { dos: 100, portsHighEdge: 3, portsLowEdge: 1 }, +}; +// 25 counted passes, 5 s apart, reach the threshold of 100 +const DOS_WITHIN_MS = 4 * 60 * 1000; +// the harness's longest removal retry (8 s) and one pass +const RELEASED_WITHIN_MS = 90 * 1000; +if (DOS_WITHIN_MS >= PASSED_WAIT_MS || RELEASED_WITHIN_MS >= DOS_WAIT_MS) { + throw new Error('3003: each budget must end before the wait only a wake can cut short'); +} +const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; +const LISTEN_PORT = 8080; + +const dosOf = async (client) => (await client.getDOSState()).data ?? {}; + +describe('a node whose router refuses it is out of service until it stops', function () { + let env; + dumpLogsOnFailure(() => env); + let node; + let nodeIp; + let appName; + let appPort; + const gateway = () => upnpGatewayState(nodeIp); + const testMappings = (requests, sinceMs = 0) => requests.filter((r) => r.action === 'AddPortMapping' + && r.description === 'Flux_Test_App' && Date.parse(r.at) >= sinceMs); + const buildApp = (name) => buildSeedableApp({ + name, + compose: [{ + name, + description: 'holds a port on the router', + repotag: `${REGISTRY_REPO_HOST}/e2eupnprefuse:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [LISTEN_PORT], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, + nodeConfigOverrides: { [NODE]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } } }, + }); + node = env.clients[NODE]; + nodeIp = subnet.nodeIp(NODE + 1); + // the first boot writes userconfig.js (its identity); a node with an api port + // in it verifies UPnP when it starts + await waitForBootSettled(node); + await enableUpnpGateway(nodeIp); + await setNodeApiPort(node.container, 16127); + await restartFluxos(node.container); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await pushBusybox('e2eupnprefuse', 'v1', 'busybox', { + entrypoint: ['/bin/busybox', 'nc', '-lk', '-p', String(LISTEN_PORT), '-e', '/bin/busybox', 'echo', 'ok'], + }); + + const { requests } = await gateway(); + expect(requests.some((r) => r.action === 'AddPortMapping' && r.description === 'Flux_UPNP_Mapping_Test'), 'canary: the node verified UPnP on its gateway').to.equal(true); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('tests its port over one protocol a pass, TCP first, and waits the long wait after a test that passed', async function () { + this.timeout(300000); + let state = { requests: [] }; + await waitFor(async () => { + state = await gateway(); + const [first] = testMappings(state.requests); + return Boolean(first) && state.requests.some((r) => r.action === 'DeletePortMapping' + && r.port === first.port && Date.parse(r.at) >= Date.parse(first.at)); + }, { timeout: 240000, interval: 2000, label: 'a test mapping made and removed' }); + + const made = testMappings(state.requests); + const [first] = made; + expect(made.filter((r) => r.port === first.port).map((r) => r.protocol), 'one protocol, TCP') + .to.deep.equal(['TCP']); + + // a failed test is retried within seconds; a passed one only after the long wait + const quietFrom = Date.now(); + await new Promise((resolve) => { setTimeout(resolve, 30000); }); + expect(testMappings((await gateway()).requests, quietFrom), 'no test mapping in the long wait').to.have.length(0); + expect((await dosOf(node)).dosMessage ?? null).to.not.equal(UPNP_DOS); + }); + + it('is out of service, with no new test mapping, once its router has refused an app\'s removal for as many passes as the threshold takes', async function () { + this.timeout(DOS_WITHIN_MS + 400000); + appName = `e2eupnprefuse${Date.now()}`; + const app = await buildApp(appName); + [appPort] = app.spec.compose[0].ports; + const mark = node.getLastEventId(); + await seedSpawnerApp(env, app, [NODE]); + await waitForAppInstalled(node, appName, 240000, { afterId: mark }); + expect((await gateway()).mappings.filter((m) => m.externalPort === appPort).map((m) => m.protocol).sort(), 'the app\'s port mapped').to.deep.equal(['TCP', 'UDP']); + + await setUpnpRefuseDeletes(nodeIp, Infinity); + const refusedFrom = Date.now(); + const auth = await authenticate(node.url, fluxTeamKey()); + const res = await fetch(`${node.url}/apps/appremove/${appName}`, { headers: { zelidauth: auth.zelidauth } }); + await res.text(); + await waitForAppRemoved(node, appName, 120000); + + await waitFor(async () => (await dosOf(node)).dosMessage === UPNP_DOS, { + timeout: DOS_WITHIN_MS, interval: 2000, label: 'the node out of service for its router', + }); + + const { requests, mappings } = await gateway(); + const refusedRemovals = requests.filter((r) => r.action === 'DeletePortMapping' && r.port === appPort && r.refused === 501); + expect(refusedRemovals.length, 'canary: the router refused the app\'s removal').to.be.at.least(2); + expect(mappings.some((m) => m.externalPort === appPort), 'the app\'s mapping still on the router').to.equal(true); + expect(testMappings(requests, refusedFrom), 'no test mapping while a removal is refused').to.have.length(0); + }); + + it('is back in service soon after its router removes the mapping, sooner than its wait in DOS', async function () { + this.timeout(RELEASED_WITHIN_MS + 60000); + const takenFrom = Date.now(); + await setUpnpRefuseDeletes(nodeIp, 0); + + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { + timeout: RELEASED_WITHIN_MS, interval: 2000, label: 'the node back in service', + }); + + const { requests, mappings } = await gateway(); + expect(mappings.some((m) => m.externalPort === appPort), 'the app\'s mapping removed').to.equal(false); + expect(testMappings(requests, takenFrom).length, 'a test ran once the router took the removal').to.be.at.least(1); + }); + + it('is out of service once a full router refuses an app\'s mapping, sooner than its wait after a passing test', async function () { + this.timeout(DOS_WITHIN_MS + 400000); + // the pass that released the node passed its test: the check is in its wait after one + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { timeout: 30000, interval: 2000 }); + const { mappings } = await gateway(); + await setUpnpCapacity(nodeIp, mappings.length); + const fullFrom = Date.now(); + const name = `e2eupnpfull${Date.now()}`; + const app = await buildApp(name); + const [port] = app.spec.compose[0].ports; + await seedSpawnerApp(env, app, [NODE]); + + await waitFor(async () => (await dosOf(node)).dosMessage === UPNP_DOS, { + timeout: DOS_WITHIN_MS, interval: 2000, label: 'the node out of service for its full router', + }); + + const { requests } = await gateway(); + const refused = requests.filter((r) => r.action === 'AddPortMapping' && r.refused === 728 && Date.parse(r.at) >= fullFrom); + expect(refused.some((r) => r.port === port), 'canary: the router refused the app\'s mapping').to.equal(true); + expect(refused.some((r) => r.description === 'Flux_Test_App'), 'the check\'s own test mapping refused too').to.equal(true); + }); + + it('is back in service at its next pass once the router has room', async function () { + this.timeout(DOS_WAIT_MS + 120000); + const roomFrom = Date.now(); + await setUpnpCapacity(nodeIp, null); + + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { + timeout: DOS_WAIT_MS + 60000, interval: 2000, label: 'the node back in service', + }); + + const made = testMappings((await gateway()).requests, roomFrom); + expect(made.some((r) => r.refused === undefined), 'a test mapping the router took').to.equal(true); + }); + + it('stays in service when its router carries out an app\'s removal but answers it with a fault', async function () { + this.timeout(DOS_WITHIN_MS + 400000); + const name = `e2eupnpfault${Date.now()}`; + const app = await buildApp(name); + const [port] = app.spec.compose[0].ports; + const mark = node.getLastEventId(); + await seedSpawnerApp(env, app, [NODE]); + await waitForAppInstalled(node, name, 240000, { afterId: mark }); + expect((await gateway()).mappings.some((m) => m.externalPort === port), 'the app\'s port mapped').to.equal(true); + + await setUpnpDeleteFault(nodeIp, 501); + const faultFrom = Date.now(); + const auth = await authenticate(node.url, fluxTeamKey()); + const res = await fetch(`${node.url}/apps/appremove/${name}`, { headers: { zelidauth: auth.zelidauth } }); + await res.text(); + await waitForAppRemoved(node, name, 120000); + + // a pending removal would count every 5 s pass and reach the DOS inside this window + const watchedUntil = Date.now() + DOS_WITHIN_MS; + while (Date.now() < watchedUntil) { + // eslint-disable-next-line no-await-in-loop + expect((await dosOf(node)).dosMessage ?? null, 'never out of service').to.not.equal(UPNP_DOS); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 5000); }); + } + await setUpnpDeleteFault(nodeIp, null); + + const { requests, mappings } = await gateway(); + const ofPort = requests.filter((r) => r.port === port && Date.parse(r.at) >= faultFrom); + expect(ofPort.some((r) => r.action === 'DeletePortMapping' && r.refused === 501), 'canary: the router answered the removal with a fault').to.equal(true); + expect(mappings.some((m) => m.externalPort === port), 'canary: and carried it out').to.equal(false); + const lookup = ofPort.findIndex((r) => r.action === 'GetSpecificPortMappingEntry'); + expect(lookup, 'the retry looked the entry up').to.be.at.least(0); + expect(ofPort.slice(lookup).filter((r) => r.action === 'DeletePortMapping'), 'no removal sent once it was found gone').to.have.length(0); + }); + + it('counts a mapping as its own while the router answers its lookup with a fault, until the router removes it', async function () { + this.timeout(DOS_WITHIN_MS + RELEASED_WITHIN_MS + 400000); + const name = `e2eupnplookup${Date.now()}`; + const app = await buildApp(name); + const [port] = app.spec.compose[0].ports; + const mark = node.getLastEventId(); + await seedSpawnerApp(env, app, [NODE]); + await waitForAppInstalled(node, name, 240000, { afterId: mark }); + expect((await gateway()).mappings.some((m) => m.externalPort === port), 'the app\'s port mapped').to.equal(true); + + await setUpnpLookupFault(nodeIp, 501); + await setUpnpRefuseDeletes(nodeIp, Infinity); + const faultFrom = Date.now(); + const auth = await authenticate(node.url, fluxTeamKey()); + const res = await fetch(`${node.url}/apps/appremove/${name}`, { headers: { zelidauth: auth.zelidauth } }); + await res.text(); + await waitForAppRemoved(node, name, 120000); + + await waitFor(async () => (await dosOf(node)).dosMessage === UPNP_DOS, { + timeout: DOS_WITHIN_MS, interval: 2000, label: 'the node out of service while the router keeps the mapping', + }); + let state = await gateway(); + const ofPort = () => state.requests.filter((r) => r.port === port && Date.parse(r.at) >= faultFrom); + expect(ofPort().some((r) => r.action === 'GetSpecificPortMappingEntry' && r.refused === 501), 'canary: a lookup answered with a fault').to.equal(true); + const firstLookup = ofPort().findIndex((r) => r.action === 'GetSpecificPortMappingEntry'); + expect(ofPort().slice(firstLookup).some((r) => r.action === 'DeletePortMapping'), 'the removal sent after a lookup that settled nothing').to.equal(true); + expect(state.mappings.some((m) => m.externalPort === port), 'the mapping still on the router').to.equal(true); + + await setUpnpRefuseDeletes(nodeIp, 0); + await waitFor(async () => (await dosOf(node)).dosMessage !== UPNP_DOS, { + timeout: RELEASED_WITHIN_MS, interval: 2000, label: 'the node back in service once the router removed the mapping', + }); + await setUpnpLookupFault(nodeIp, null); + state = await gateway(); + expect(state.mappings.some((m) => m.externalPort === port), 'the mapping removed').to.equal(false); + }); +}); diff --git a/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js new file mode 100644 index 0000000000..2170daa098 --- /dev/null +++ b/test-infra/runner/tests/3004-a-port-is-proven-over-udp.js @@ -0,0 +1,449 @@ +// weight: heavy +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { getSubnetConfig, REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { execInContainer, restartFluxos, setNodeApiPort } from '../framework/container.js'; +import { enableUpnpGateway, setUpnpProtocolFault, upnpGatewayState } from '../framework/external-http-control.js'; +import { waitFor, waitForAppInstalled, waitForBootSettled } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// Every app port is published for TCP and UDP, so a node proves both before it +// installs an app, and its availability check proves both on a port of its own. +// +// UDP has no handshake: silence is all a closed port and a lost datagram say. A +// peer probes each port and hands back what replied; only the node's own token +// coming back proves a port, and a refusal needs two distinct peers that heard +// nothing. The shapes put the fault in the node's own namespace, on its fleet +// device: +// +// udp-blocked no datagram starting a flow arrives - a router forwarding TCP +// alone. Refused, and the availability check reports it. +// udp-first-lost the first datagram from each source is lost, every later one +// arrives. A probe that sends once never gets through; one that +// sends again always does. Proven. +// udp-every-other every other datagram is lost, whatever its source. Proven. +// udp-replies-lost on a PEER: it never hears a reply to its own probes, so it is +// silent about every node. Two such peers do not refuse a node +// while a peer that proves its port is still to ask. +// +// The open node runs ufw as an install leaves it, so its UDP test rule is what +// admits the probes, and it is gone once the test ends. The every-other node is +// behind a UPnP gateway, which must be asked for both protocols of the port. +// +// The peers in the first two fleets are real nodes on this branch. The last +// fleet's peers are stubs that read TCP and do not probe UDP - the state of the +// network while it updates - and a node there must install on TCP proof alone. + +const subnet = getSubnetConfig(); +const socketAddressOf = (index) => `${subnet.nodeIp(index + 1)}:16127`; + +// The availability check's waits, compressed: a pass after a failure comes a +// second later, and a confirmed failure takes two passes. +const APPS_PORT_CHECK = { + timeoutsMs: { + default: 10 * 60 * 1000, error: 5000, failure: 1000, dos: 5 * 60 * 1000, appError: 5000, + }, + thresholds: { dos: 100, portsHighEdge: 3, portsLowEdge: 1 }, +}; + +// Enough attempts that a node always reaches two peers that answer. +const PORT_TEST = { portTestMaxAttempts: 3 }; + +const PORTS_DOS_PREFIX = 'Ports tested not reachable from outside'; +const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; + +const appPinnedTo = async (name, port, index) => { + await pushImage(name, 'v1'); + return buildSeedableApp({ + name, + instances: 1, + nodes: [socketAddressOf(index)], + compose: [{ + name, + description: 'one port, proven before install', + repotag: `${REGISTRY_REPO_HOST}/${name}:v1`, + ports: [port], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); +}; + +// A fit node answers with its checks as the data; an unfit one, under data.checks. +const udpCheck = async (client) => { + const health = await client.get('/flux/health').catch(() => null); + return health?.status === 'success' ? health.data?.udp : health?.data?.checks?.udp; +}; + +// Seeds an app pinned to one node and waits for that node's port test to prove +// its port, then for the install. +const provenAndInstalled = async (env, index, name, port) => { + const client = env.clients[index]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo(name, port, index)); + const proven = await client.waitForEvent('ports:proven', (d) => d.ports.includes(port), 420000, { afterId: after }); + expect(proven.data.reason).to.equal('proven'); + await waitForAppInstalled(client, name, 300000, { afterId: after }); +}; + +describe('a port is proven over UDP', function () { + const OPEN = 0; + const BLOCKED = 1; + const FIRST_LOST = 2; + const EVERY_OTHER = 3; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + firewall: [OPEN], + networkShapes: { [BLOCKED]: 'udp-blocked', [FIRST_LOST]: 'udp-first-lost', [EVERY_OTHER]: 'udp-every-other' }, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + nodeConfigOverrides: { + [OPEN]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } }, + [BLOCKED]: { fluxapps: { appsPortCheck: APPS_PORT_CHECK } }, + }, + }); + // the first boot writes userconfig.js (its identity); a node with an api port + // in it verifies UPnP when it starts + const upnp = env.clients[EVERY_OTHER]; + await waitForBootSettled(upnp); + await enableUpnpGateway(subnet.nodeIp(EVERY_OTHER + 1)); + await setNodeApiPort(upnp.container, 16127); + await restartFluxos(upnp.container); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('installs on a firewalled node whose port answers over TCP and UDP, and leaves no test rule behind', async function () { + this.timeout(720000); + await provenAndInstalled(env, OPEN, 'udpopenapp', 31141); + + const rules = (await execInContainer(env.clients[OPEN].container, 'ufw status; true')).stdout + .split('\n').map((line) => line.trim()).filter((line) => line.startsWith('31141')); + expect(rules.some((line) => /^31141\s/.test(line)), 'canary: the app\'s own rule for its port').to.equal(true); + expect(rules.filter((line) => line.includes('/udp') || line.includes('/tcp')), 'a test rule left behind').to.deep.equal([]); + }); + + it('refuses a port two peers hear nothing on over UDP', async function () { + this.timeout(420000); + const client = env.clients[BLOCKED]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo('udpblockedapp', 31142, BLOCKED)); + + // Every peer is silent here, and the node may ask as many as it has + // attempts. Silence is weighed only once no attempt is left that could prove + // the port, so the refusal names a peer for every attempt. + const refused = await client.waitForEvent('ports:udpUnreachable', (d) => d.port === 31142, 420000, { afterId: after }); + expect(refused.data.reason).to.equal('udpUnreachable'); + expect(refused.data.peers, 'refused before every attempt was spent').to.have.length(PORT_TEST.portTestMaxAttempts); + expect(new Set(refused.data.peers).size).to.equal(PORT_TEST.portTestMaxAttempts); + await client.waitForEvent( + 'spawner:deferred', + (d) => d.appName === 'udpblockedapp' && d.reason === 'ports_not_available', + 60000, + { afterId: after }, + ); + const installed = await client.getInstalledApps(); + expect(installed.data.map((a) => a.name)).to.not.include('udpblockedapp'); + }); + + it('proves a port whose first datagram from each peer is lost', async function () { + this.timeout(720000); + await provenAndInstalled(env, FIRST_LOST, 'udplossyapp', 31143); + }); + + it('proves a port that loses every other datagram, behind UPnP mapped for both protocols', async function () { + this.timeout(720000); + await provenAndInstalled(env, EVERY_OTHER, 'udpsteadylossapp', 31145); + + const { requests } = await upnpGatewayState(subnet.nodeIp(EVERY_OTHER + 1)); + const mapped = requests.filter((r) => r.action === 'AddPortMapping' && r.port === 31145 && r.description === 'Flux_App_udpsteadylossapp'); + expect([...new Set(mapped.map((r) => r.protocol))].sort()).to.deep.equal(['TCP', 'UDP']); + }); + + // Two peers' silence is reported in health. Taking the node out of service for + // it waits on a policy document this fleet's policy does not carry. + it('reports UDP degraded in health, and stays in service, when the availability check confirms the failure', async function () { + this.timeout(600000); + await waitFor(async () => (await udpCheck(env.clients[BLOCKED])) === 'degraded', { + timeout: 540000, interval: 5000, label: 'the blocked node reports udp degraded', + }); + await waitFor(async () => (await udpCheck(env.clients[OPEN])) === 'ok', { + timeout: 60000, interval: 5000, label: 'the firewalled open node reports udp ok', + }); + const dos = (await env.clients[BLOCKED].getDOSState()).data; + expect(dos.dosMessage).to.equal(null); + }); +}); + +describe('a node whose first peers cannot hear UDP replies', function () { + const NODE = 0; + const DEAF = [1, 2]; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: Object.fromEntries(DEAF.map((i) => [i, 'udp-replies-lost'])), + // every other node is asked, the two deaf ones among them + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, portTestMaxAttempts: 3 } }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('is proven by the peer that hears it', async function () { + this.timeout(720000); + await provenAndInstalled(env, NODE, 'udpdeafpeersapp', 31146); + }); +}); + +// A node behind UPnP maps one protocol a pass, TCP and UDP in turn, so a pass +// costs its router one mapping and one removal. A router refusing the UDP +// mapping is as broken as one refusing TCP. +describe('a node behind UPnP tests one protocol a pass', function () { + const NODE = 0; + let env; + let nodeIp; + dumpLogsOnFailure(() => env); + const testMappings = (requests) => requests.filter((r) => r.action === 'AddPortMapping' && r.description === 'Flux_Test_App'); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + // a passing test is followed by the next pass 20 s later, and a router fault + // reaches the DOS on its second count + nodeConfigOverrides: { + [NODE]: { + fluxapps: { + appsPortCheck: { + timeoutsMs: { + default: 20000, error: 2000, failure: 1000, dos: 5 * 60 * 1000, appError: 5000, + }, + thresholds: { dos: 8, portsHighEdge: 3, portsLowEdge: 1 }, + }, + }, + }, + }, + }); + nodeIp = subnet.nodeIp(NODE + 1); + const node = env.clients[NODE]; + await waitForBootSettled(node); + await enableUpnpGateway(nodeIp); + await setNodeApiPort(node.container, 16127); + await restartFluxos(node.container); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('maps TCP and UDP on alternate passes, one mapping a pass, and proves both', async function () { + this.timeout(600000); + let made = []; + await waitFor(async () => { + made = testMappings((await upnpGatewayState(nodeIp)).requests); + return made.length >= 3; + }, { timeout: 480000, interval: 5000, label: 'three passes behind UPnP' }); + + const [first, second, third] = made; + expect([first.protocol, second.protocol, third.protocol]).to.deep.equal(['TCP', 'UDP', 'TCP']); + expect(new Set([first.port, second.port, third.port]).size, 'one mapping a pass').to.equal(3); + expect(await udpCheck(env.clients[NODE])).to.equal('ok'); + }); + + it('is out of service, mapping UDP and nothing else, once its router refuses UDP mappings', async function () { + this.timeout(600000); + await setUpnpProtocolFault(nodeIp, 'UDP', 606); + const from = Date.now(); + + await waitFor(async () => (await env.clients[NODE].getDOSState()).data?.dosMessage === UPNP_DOS, { + timeout: 480000, interval: 5000, label: 'the node is out of service for its router', + }); + + const refusedSince = testMappings((await upnpGatewayState(nodeIp)).requests).filter((r) => Date.parse(r.at) >= from); + const lastTcp = refusedSince.findLastIndex((r) => r.protocol === 'TCP'); + const firstRefusal = refusedSince.findIndex((r) => r.refused === 606); + expect(firstRefusal, 'canary: the router refused a UDP mapping').to.be.at.least(0); + expect(lastTcp, 'a TCP pass after the router refused UDP').to.be.below(firstRefusal); + await setUpnpProtocolFault(nodeIp, 'UDP', null); + }); +}); + +// A pass whose TCP passes clears what the TCP failures before it left, even +// when the first peer hears nothing over UDP and the port is retested over UDP. +describe('a node whose TCP comes back while its UDP is blocked', function () { + const NODE = 0; + let env; + dumpLogsOnFailure(() => env); + // Inbound TCP connections to the node, bar its FluxOS ports, dropped on its fleet device. + const tcpBlock = (op) => { + const ip = subnet.nodeIp(NODE + 1); + return `dev=$(ip -o -4 addr show | awk -v ip=${ip} '{ split($4, a, "/"); if (a[1] == ip) print $2 }'); ` + + `iptables -w ${op} INPUT -i "$dev" -p tcp -m conntrack --ctstate NEW -m multiport ! --dports 16100:16299 -j DROP`; + }; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: { [NODE]: 'udp-blocked' }, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + // failed ports never outnumber the low edge, so a passing pass clears them all + nodeConfigOverrides: { + [NODE]: { fluxapps: { appsPortCheck: { ...APPS_PORT_CHECK, thresholds: { dos: 100, portsHighEdge: 2, portsLowEdge: 2 } } } }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + // Which branch a pass took is a tally on /flux/testcounters, never a stream. + it('clears its failed ports on the pass whose TCP passes and whose first peer hears nothing over UDP', async function () { + this.timeout(600000); + const client = env.clients[NODE]; + const tally = async (name, ...path) => path.reduce((node, key) => node?.[key], (await client.getTestCounters())[name]) || 0; + + const blocked = await execInContainer(client.container, tcpBlock('-I')); + expect(blocked.exitCode, blocked.output).to.equal(0); + const unreachableBefore = await tally('availability:pass', 'unreachable', 'unknown'); + await waitFor(async () => (await tally('availability:pass', 'unreachable', 'unknown')) > unreachableBefore, { + timeout: 300000, interval: 2000, label: 'a pass whose TCP test failed', + }); + + const unblocked = await execInContainer(client.container, tcpBlock('-D')); + expect(unblocked.exitCode, unblocked.output).to.equal(0); + await waitFor(async () => (await tally('availability:firstUdpSilence', 'passed', 'cleared')) >= 1 + || (await tally('availability:firstUdpSilence', 'passed', 'notCleared')) >= 1, { + timeout: 300000, interval: 2000, label: 'a pass whose TCP passed and whose first peer heard nothing over UDP', + }); + + expect(await tally('availability:firstUdpSilence', 'passed', 'notCleared'), 'failed ports left for the retest to clear').to.equal(0); + expect(await tally('availability:firstUdpSilence', 'passed', 'cleared')).to.be.at.least(1); + }); +}); + +describe('a port closed over UDP, under the policy that counts it and turns the install refusal off', function () { + const BLOCKED = 0; + // blocked too, on the default thresholds, so it stays in service to install + const INSTALLS = 1; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + networkShapes: { [BLOCKED]: 'udp-blocked', [INSTALLS]: 'udp-blocked' }, + policy: { documents: { udpportdos: true, udpinstallrefusaloff: true } }, + configOverrides: { fluxapps: { minOutgoing: 2, minIncoming: 1, ...PORT_TEST } }, + // the first port failure is walked, the next counts, and one count is the threshold + nodeConfigOverrides: { + [BLOCKED]: { fluxapps: { appsPortCheck: { ...APPS_PORT_CHECK, thresholds: { dos: 4, portsHighEdge: 1, portsLowEdge: 0 } } } }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + // The switch that turns the refusal off without a release. + it('installs on a port two peers hear nothing on over UDP, as udpUnproven', async function () { + this.timeout(420000); + const client = env.clients[INSTALLS]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo('udprefusaloffapp', 31147, INSTALLS)); + + const unproven = await client.waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); + expect(unproven.data.reason).to.equal('udpUnproven'); + expect(unproven.data.peers, 'canary: peers heard nothing over UDP').to.have.length.at.least(2); + await waitForAppInstalled(client, 'udprefusaloffapp', 300000, { afterId: after }); + }); + + it('takes the node out of service once the failures it confirms reach the threshold', async function () { + this.timeout(600000); + await waitFor(async () => (await env.clients[BLOCKED].getDOSState()).data?.dosMessage?.startsWith(PORTS_DOS_PREFIX) === true, { + timeout: 540000, interval: 5000, label: 'the blocked node is out of service for its ports', + }); + expect(await udpCheck(env.clients[BLOCKED])).to.equal('degraded'); + }); +}); + +describe('a node whose peers do not probe UDP', function () { + const NODE = 0; + let env; + dumpLogsOnFailure(() => env); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: 4, + stubPeers: [1, 2, 3], + networkShapes: { [NODE]: 'udp-blocked' }, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1, ...PORT_TEST } }, + }); + await bootAndPeer(env); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + // The first nodes to update have almost no peer that probes UDP. Silence they + // cannot get is not silence, so they install on TCP proof as before. + it('installs on TCP proof, its UDP unproven', async function () { + this.timeout(420000); + const client = env.clients[NODE]; + const after = client.getLastEventId(); + await seedSpawnerApp(env, await appPinnedTo('udpoldpeersapp', 31144, NODE)); + + const unproven = await client.waitForEvent('ports:unproven', () => true, 420000, { afterId: after }); + expect(unproven.data.reason).to.equal('udpUnproven'); + expect(unproven.data.peers).to.deep.equal([]); + await waitForAppInstalled(client, 'udpoldpeersapp', 300000, { afterId: after }); + }); +}); diff --git a/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js new file mode 100644 index 0000000000..d1ed60ecc4 --- /dev/null +++ b/test-infra/runner/tests/3301-a-database-that-stops-answering-takes-nothing-down.js @@ -0,0 +1,234 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { + execInContainer, getAppContainerStatus, crashFluxos, releaseFluxos, fluxosPid, +} from '../framework/container.js'; +import { + waitFor, waitForUp, waitForDown, waitForOperatorIntent, waitForBootSettled, assertNoEvent, +} from '../framework/wait.js'; +import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; +import { holdCommands, releaseCommands } from '../framework/mongo-control.js'; +import { getSubnetConfig } from '../framework/subnet-config.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A mongod that stops answering FluxOS's commands while it goes on answering the +// driver's heartbeats, and then answers again: what a mongod with one thread stuck +// in the kernel looks like from FluxOS (mongo-control). The fleet shares one mongod. +// Every call waits and none fails, and only a bound on the call ends the wait. +// FluxOS bounds a call at config.database.callTimeoutMs, and a call on the client +// for long operations at maintenanceCallTimeoutMs; a read is retried once. Every +// caller of a call that fails leaves what it cannot read as it is. This fleet runs +// with both bounds short, and held for longer than a read's whole budget, it: +// +// - answers a call that reads the database, with an error, rather than waiting +// for the database: what turns a stall into a failure each caller handles; +// - keeps every FluxOS process it had, so nothing exits on a failed call; +// - removes no app and announces no removal; +// - keeps an operator-stopped component stopped; +// - announces its apps again once mongod answers. +// +// The second case holds the commands while a node boots. The boot's schema step +// runs on the client for long operations and fails each time its bound runs out; +// boot runs that step again until it succeeds, so the boot runs once rather than +// starting again from the top. + +const subnet = getSubnetConfig(); +const CALL_TIMEOUT_MS = 10 * 1000; +const MAINTENANCE_CALL_TIMEOUT_MS = 20 * 1000; +// A read that has to wait for a pooled connection first is answered within one +// bound for the pool and two for the read and its retry. +const READ_ANSWERED_WITHIN_MS = 3 * CALL_TIMEOUT_MS + 10 * 1000; +// Over a read's whole budget with room to tell it from the hold, and over three +// of the schema step's bounds. +const HOLD_MS = 75 * 1000; +// The driver's own words for a round trip and for a pooled-connection wait that ran +// out, in a whole error and in its message alike, and nothing else logs them. One of +// them shows the calls failed, so the assertions below are about failed calls. +const DATABASE_GAVE_NO_ANSWER = /connection \d+ to \S+ timed out|Timed out while checking out a connection from connection pool/; + +// A TAP comment, so a hold that overruns says which step it was in. +function step(what) { + console.log(`# 3301 ${new Date().toISOString()} ${what}`); +} + +describe('a database that stops answering takes nothing down', function () { + let env; + dumpLogsOnFailure(() => env); + const runningName = `e2edbrun${Date.now()}`; + const stoppedName = `e2edbstop${Date.now()}`; + let runningIdx; + let stoppedIdx; + let held = false; + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ + hookCtx: this, + nodes: 10, + tickerAutostart: false, + configOverrides: { database: { callTimeoutMs: CALL_TIMEOUT_MS, maintenanceCallTimeoutMs: MAINTENANCE_CALL_TIMEOUT_MS } }, + }); + await bootAndPeer(env); + ({ index: runningIdx } = await seedSimpleApp(env, runningName)); + ({ index: stoppedIdx } = await seedSimpleApp(env, stoppedName)); + await waitForUp(env.clients[runningIdx], runningName, 'the running app is up'); + await waitForUp(env.clients[stoppedIdx], stoppedName, 'the app to stop is up'); + + const client = env.clients[stoppedIdx]; + const auth = await authenticate(client.url, appOwnerKey()); + const stopRes = await client.getAuthed(`/apps/appstop/${stoppedName}`, auth.zelidauth); + expect(stopRes.status, `appstop refused: ${JSON.stringify(stopRes.data)}`).to.equal('success'); + await waitForOperatorIntent(client, `${stoppedName}_${stoppedName}`, true); + await waitForDown(client, stoppedName, 'stopped by its owner'); + }); + + after(async function () { + this.timeout(60000); + if (held) await releaseCommands().catch(() => {}); + await env?.teardown(); + }); + + describe('while the fleet runs', function () { + let pidsBefore; + let anchors; + let read; + + before(async function () { + this.timeout(HOLD_MS + 120000); + step('reading the FluxOS pids'); + pidsBefore = await Promise.all(env.clients.map((c) => fluxosPid(c.container))); + anchors = env.clients.map((client) => client.getLastEventId()); + step('holding the commands'); + await holdCommands(HOLD_MS); + held = true; + step('held'); + try { + await new Promise((resolve) => { setTimeout(resolve, 5000); }); + const startedAt = Date.now(); + const reading = env.clients[runningIdx].get(`/apps/appspecifications/${runningName}`) + .then((body) => ({ body, ms: Date.now() - startedAt }), (error) => ({ error, ms: Date.now() - startedAt })); + await new Promise((resolve) => { setTimeout(resolve, HOLD_MS - 5000); }); + read = await Promise.race([reading, { ms: null }]); + if (read.ms === null) read = { ms: Date.now() - startedAt, unanswered: true }; + step(`read ${read.unanswered ? 'unanswered' : 'answered'} after ${read.ms} ms`); + } finally { + step('releasing the commands'); + await releaseCommands(); + held = false; + step('released'); + } + }); + + it('answers a call that reads the database, rather than waiting for it', function () { + expect(read.unanswered, `the node gave no answer in ${Math.round(read.ms / 1000)} s while the database gave none`).to.not.equal(true); + expect(read.ms, 'answered within a read\'s budget').to.be.below(READ_ANSWERED_WITHIN_MS); + }); + + it('reached FluxOS: a node logged a call the database did not answer', function () { + const stalled = env.clients.map((_, i) => i).filter((i) => env.nodeHasLog(i, DATABASE_GAVE_NO_ANSWER)); + expect(stalled, 'no node logged a database call that timed out, so nothing below was tested').to.not.be.empty; + }); + + it('keeps every FluxOS process it had', async function () { + const pidsAfter = await Promise.all(env.clients.map((c) => fluxosPid(c.container))); + expect(pidsAfter, 'a node\'s FluxOS exited during the hold').to.deep.equal(pidsBefore); + }); + + it('removes no app and announces no removal', async function () { + this.timeout(30000); + await Promise.all(env.clients.map(async (client, i) => { + await assertNoEvent(client, 'app:removed', () => true, 1, { afterId: anchors[i] }); + await assertNoEvent(client, 'network:appremoved', () => true, 1, { afterId: anchors[i] }); + })); + const running = await getAppContainerStatus(env.clients[runningIdx].container, runningName); + expect(running && running.status.startsWith('Up'), 'the running app is still up').to.equal(true); + }); + + it('keeps the operator-stopped component stopped', async function () { + this.timeout(30000); + const client = env.clients[stoppedIdx]; + await assertNoEvent(client, 'reconciler:actuated', (d) => d.identifier === `${stoppedName}_${stoppedName}` && d.action === 'started', 1, { afterId: anchors[stoppedIdx] }); + const stopped = await getAppContainerStatus(client.container, stoppedName, { all: true }); + expect(stopped && stopped.status.startsWith('Up'), 'the stopped component was started').to.not.equal(true); + }); + + it('announces its apps again once the database answers', async function () { + this.timeout(240000); + const holderIp = subnet.nodeIp(runningIdx + 1); + const peer = env.clients[(runningIdx + 1) % env.clients.length]; + const afterId = peer.getLastEventId(); + await peer.waitForEvent( + 'network:apprunning', + (d) => d.ip?.startsWith(holderIp) && d.apps?.some((a) => a.name === runningName), + 200000, + { afterId }, + ); + const installed = await env.clients[runningIdx].get('/apps/installedapps'); + expect(installed.status, 'the node reads its installed apps again').to.equal('success'); + expect(installed.data.map((app) => app.name)).to.include(runningName); + }); + }); + + describe('while a node boots', function () { + const idx = 0; + // One line from the stretch of boot before the schema, and one from after it: a + // boot that started again from the top would log either of them twice. + const BEFORE_SCHEMA = /System service initiated/; + const AFTER_SCHEMA = /Flux checks operational/; + // The schema phase begun and not finished when the hold ends: the boot was held + // on the database, inside the step boot runs again until it succeeds. + const SCHEMA_BEGUN = /Preparing local database/; + const SCHEMA_FINISHED = /Local database prepared/; + let counts; + let atRelease; + let bootAnchor; + + before(async function () { + this.timeout(HOLD_MS + 300000); + const { container } = env.clients[idx]; + // Event ids are seeded from the clock and keep rising across a restart. + bootAnchor = env.clients[idx].getLastEventId(); + await crashFluxos(container, { hold: true }); + counts = { + before: env.nodeLogCount(idx, BEFORE_SCHEMA), + after: env.nodeLogCount(idx, AFTER_SCHEMA), + begun: env.nodeLogCount(idx, SCHEMA_BEGUN), + finished: env.nodeLogCount(idx, SCHEMA_FINISHED), + noAnswer: env.nodeLogCount(idx, DATABASE_GAVE_NO_ANSWER), + }; + await holdCommands(HOLD_MS); + held = true; + try { + await releaseFluxos(container); + await new Promise((resolve) => { setTimeout(resolve, HOLD_MS); }); + atRelease = { + begun: env.nodeLogCount(idx, SCHEMA_BEGUN) - counts.begun, + finished: env.nodeLogCount(idx, SCHEMA_FINISHED) - counts.finished, + noAnswer: env.nodeLogCount(idx, DATABASE_GAVE_NO_ANSWER) - counts.noAnswer, + }; + } finally { + await releaseCommands(); + held = false; + } + }); + + it('held the boot inside its schema step while the database gave no answer', function () { + expect(atRelease.begun, 'the boot never reached its schema step, so nothing below was tested').to.be.above(0); + expect(atRelease.noAnswer, 'no call of the boot failed while it was held, so nothing below was tested').to.be.above(0); + expect(atRelease.finished, 'the schema step finished while the database held every command').to.equal(0); + }); + + it('finishes the boot once, after the database answers', async function () { + this.timeout(240000); + await waitForBootSettled(env.clients[idx], 200000, { afterId: bootAnchor }); + await waitFor(() => env.nodeLogCount(idx, AFTER_SCHEMA) > counts.after, { + timeout: 60000, interval: 2000, label: 'the boot reaches the start of the flux checks', + }); + expect(env.nodeLogCount(idx, BEFORE_SCHEMA) - counts.before, 'the stretch before the schema ran').to.equal(1); + expect(env.nodeLogCount(idx, AFTER_SCHEMA) - counts.after, 'the start phase ran').to.equal(1); + }); + }); +}); diff --git a/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js new file mode 100644 index 0000000000..e87504283c --- /dev/null +++ b/test-infra/runner/tests/3302-a-database-that-keeps-timing-out-takes-on-no-new-apps.js @@ -0,0 +1,106 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { getAppContainerStatus } from '../framework/container.js'; +import { waitForUp, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; +import { holdCommands, releaseCommands } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A mongod that keeps leaving FluxOS's calls unanswered, held with mongo-control as +// suite 3301 holds it. Once suspectRoundTrips round trips have run out within +// suspectWindowMs, the node holds itself back from new placements, and it lifts the +// hold suspectWindowMs after the last. The apps it runs stay: heavy load also makes +// calls time out. + +const CALL_TIMEOUT_MS = 5 * 1000; +const SUSPECT_WINDOW_MS = 60 * 1000; +// Long enough for many round trips to run out on every node. +const HOLD_MS = 45 * 1000; +const OWNER = 'databaseSuspect'; +const byOwner = (data) => data.owner === OWNER; + +function step(what) { + console.log(`# 3302 ${new Date().toISOString()} ${what}`); +} + +describe('a database that keeps timing out takes on no new apps', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2edbsuspect${Date.now()}`; + let appIdx; + let anchors; + let held = false; + let releasedAt; + let heldFrom = {}; + + // Each node that held itself back during the stall, with the event that says so. + function nodesThatHeld() { + return env.clients.map((client, i) => ({ + i, + set: client.getEventBuffer().find((e) => e.event === 'placementHold:set' && e.id > anchors[i] && byOwner(e.data)), + })).filter(({ set }) => set); + } + + before(async function () { + this.timeout(420000 + HOLD_MS); + env = await createTestEnv({ + hookCtx: this, + nodes: 6, + tickerAutostart: false, + configOverrides: { database: { callTimeoutMs: CALL_TIMEOUT_MS, suspectWindowMs: SUSPECT_WINDOW_MS } }, + }); + await bootAndPeer(env); + ({ index: appIdx } = await seedSimpleApp(env, appName)); + await waitForUp(env.clients[appIdx], appName, 'the app is up'); + + anchors = env.clients.map((client) => client.getLastEventId()); + step('holding the commands'); + await holdCommands(HOLD_MS); + held = true; + try { + await new Promise((resolve) => { setTimeout(resolve, HOLD_MS); }); + } finally { + step('releasing the commands'); + heldFrom = await releaseCommands(); + held = false; + releasedAt = Date.now(); + step('released'); + } + }); + + after(async function () { + this.timeout(60000); + if (held) await releaseCommands().catch(() => {}); + await env?.teardown(); + }); + + it('reached FluxOS: the database held the nodes\' calls unanswered', function () { + const stalled = env.clients.filter((client) => heldFrom[client.ip] > 0); + expect(stalled, `the database held no node's call (held: ${JSON.stringify(heldFrom)}), so nothing below was tested`).to.not.be.empty; + }); + + it('holds a node whose database kept timing out back from new placements', function () { + const holding = nodesThatHeld(); + expect(holding, 'no node held itself back while its database left its calls unanswered').to.not.be.empty; + }); + + it('keeps the apps it runs', async function () { + this.timeout(30000); + await Promise.all(env.clients.map(async (client, i) => { + await assertNoEvent(client, 'app:removed', () => true, 1, { afterId: anchors[i] }); + })); + const running = await getAppContainerStatus(env.clients[appIdx].container, appName); + expect(running && running.status.startsWith('Up'), 'the app is still up').to.equal(true); + }); + + it('lifts the hold a window after the last round trip that ran out', async function () { + this.timeout(SUSPECT_WINDOW_MS + 120000); + const holding = nodesThatHeld(); + expect(holding, 'no node held itself back, so there is no hold to lift').to.not.be.empty; + await Promise.all(holding.map(async ({ i, set }) => { + await env.clients[i].waitForEvent('placementHold:cleared', byOwner, SUSPECT_WINDOW_MS + 90000, { afterId: set.id }); + })); + expect(Date.now() - releasedAt, 'a hold lifted before its window could have passed').to.be.at.least(SUSPECT_WINDOW_MS - CALL_TIMEOUT_MS * 2); + }); +}); diff --git a/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js new file mode 100644 index 0000000000..e28616a053 --- /dev/null +++ b/test-infra/runner/tests/3303-a-stalled-host-takes-the-node-out-of-service.js @@ -0,0 +1,513 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, getAppContainerStatus } from '../framework/container.js'; +import { + waitFor, waitForUp, waitForAppInstalled, assertNoEvent, +} from '../framework/wait.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { buildSeedableTestApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { + freezeDocker, thawDocker, DOCKER_ROOT, + thawEveryNode, stuckThreads, processesThatCanRun, containerCgroup, + stallDisk, resumeDisk, readsHeld, +} from '../framework/host-stall.js'; +import { + DOCKERD_UNIT, freezeUnit, restartFluxos, unitCgroup, +} from '../framework/systemd-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A host that stops returning the disk under docker, on a real kernel: the +// filesystem the node's docker keeps its data on is frozen (framework/host-stall.js) +// while an app writes a log line every 200 ms, so docker's own threads are held in +// the kernel and never run - the stall FluxOS's detector reads from /proc. +// +// A thread held for one reading interval holds the node back from new placements, +// and the hold lifts when the thread runs again. Held for stuckIntervals intervals, +// the node goes out of service until it reboots: every app is asked to stop at +// once, whatever of them can still run after one shared stop window is killed +// through its slice, and then each is removed within a deadline. On a frozen disk +// docker still delivers the stop signal, so the app is stopped though its removal +// runs out. With docker frozen as well (systemd-control.freezeUnit), docker +// cannot act at all, and the app is killed without docker. A node holding several +// apps has every one of them stopped within that one window, long before even one +// removal's deadline. A node whose FluxOS restarts while the stall is under way +// carries on counting it, and goes out of service when it would have without the +// restart; one whose FluxOS restarts over and over adds nothing to the count for +// the restarts. A node whose docker disk stops answering +// reads as well, so docker's own records of its containers cannot be read, stops +// and kills its apps through their slices within the same window. +// +// The nodes run systemd, so docker uses its systemd cgroup driver and every app's +// containers live in the app's own slice, as on a production node. The node the +// kill case runs on is a legacy node whose FluxOS runs as the operator's account, +// not root: it reads docker's threads, asks systemd and kills through sudo. +// +// Each case first asserts, independently of FluxOS, that docker was held: on a tree +// without the detector that canary passes and the assertions after it fail. + +const READ_INTERVAL_MS = 5 * 1000; +const STUCK_INTERVALS = 6; +const REMOVAL_DEADLINE_MS = 45 * 1000; +const STOP_WINDOW_MS = 10 * 1000; +// what the node takes beyond the window to check and kill every app +const KILL_SLACK_MS = 10 * 1000; +const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; +const OWNER = 'hostStall'; +// the Arcane nodes the first case's app and the several apps' case share +const ARCANE = [0, 1]; +// an Arcane node whose FluxOS restarts mid-stall +const RESTARTED = 2; +const LEGACY_UNPRIVILEGED = 3; +// an Arcane node of its own: every case leaves its node out of service +const DISK_STALLED = 4; +// The node whose FluxOS restarts mid-stall reads every RESTART_READ_INTERVAL_MS, so +// the readings a restart would lose take far longer than a restart does. +const RESTART_READ_INTERVAL_MS = 20 * 1000; +// A node whose FluxOS restarts over and over while its docker is held, every +// restart well inside one reading interval, so restarts that each counted as one +// would take it out of service long before the readings could. +const CRASH_LOOPED = 5; +const CRASH_LOOP_READ_INTERVAL_MS = 180 * 1000; +// a start's reading for each restart would reach the threshold with one to spare +const CRASH_LOOP_RESTARTS = STUCK_INTERVALS + 2; + +const byOwner = (data) => data.owner === OWNER; +const outOfServiceForTheStall = (data) => data.dosState >= 100 + && String(data.dosMessage).includes('Host stall') && String(data.dosMessage).includes('dockerd'); +// the app names here have no '-', which a slice name escapes +const sliceOf = (appName) => `flux-${appName}.slice`; +const containerOf = (appName) => `flux${appName}_${appName}`; +// what of the app can still run, read from its slice, which outlives its containers +async function sliceProcessesThatCanRun(container, appName) { + return processesThatCanRun(container, await unitCgroup(container, sliceOf(appName))); +} + +function step(what) { + console.log(`# 3303 ${new Date().toISOString()} ${what}`); +} + +// An app that writes a log line every 200 ms, installed on one of `indices` only. +async function installOn(env, name, indices) { + await pushTestApp(name); + const app = await buildSeedableTestApp({ name, logEveryMs: 200, instances: 1 }); + await seedSpawnerApp(env, app, indices); + return Promise.any(indices.map(async (i) => { + await waitForAppInstalled(env.clients[i], name, 120000); + return i; + })); +} + +describe('a stalled host takes the node out of service', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2ehoststall${Date.now()}`; + let node; + let client; + let container; + + before(async function () { + this.timeout(480000); + env = await createTestEnv({ + hookCtx: this, + nodes: 6, + tickerAutostart: false, + freezableDockerRoot: [...ARCANE, RESTARTED, LEGACY_UNPRIVILEGED, DISK_STALLED, CRASH_LOOPED], + nodeConfigOverrides: { + [RESTARTED]: { hostStall: { readIntervalMs: RESTART_READ_INTERVAL_MS } }, + [CRASH_LOOPED]: { hostStall: { readIntervalMs: CRASH_LOOP_READ_INTERVAL_MS } }, + }, + legacyNodes: [LEGACY_UNPRIVILEGED], + unprivilegedNodes: [LEGACY_UNPRIVILEGED], + configOverrides: { + hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS, stopWindowMs: STOP_WINDOW_MS }, + fluxapps: { appRemovalDeadlineMs: REMOVAL_DEADLINE_MS }, + }, + }); + await bootAndPeer(env); + node = await installOn(env, appName, ARCANE); + client = env.clients[node]; + ({ container } = client); + await waitForUp(client, appName, 'the app is up'); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('runs the app in its own slice', async function () { + const cgroup = await containerCgroup(container, containerOf(appName)); + const slice = await unitCgroup(container, sliceOf(appName)).catch(() => null); + expect(slice !== null && cgroup.startsWith(`${slice}/`), `the app container is in ${cgroup}, not in its slice ${slice}, so the kill below has nothing to find`).to.equal(true); + }); + + describe('a stall shorter than the threshold', function () { + let anchor; + + before(async function () { + anchor = client.getLastEventId(); + step('freezing'); + await freezeDocker(container); + }); + + after(async function () { + await thawEveryNode([client]); + }); + + it('held docker in the kernel', async function () { + const stuck = await stuckThreads(container, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + }); + + it('holds the node back from new placements', async function () { + this.timeout(READ_INTERVAL_MS * 4); + await client.waitForEvent('placementHold:set', byOwner, READ_INTERVAL_MS * 3, { afterId: anchor }); + }); + + it('lifts the hold once docker runs again, and stays in service', async function () { + this.timeout(READ_INTERVAL_MS * 6); + step('thawing'); + await thawDocker(container); + await client.waitForEvent('placementHold:cleared', byOwner, READ_INTERVAL_MS * 4, { afterId: anchor }); + await assertNoEvent(client, 'dos:changed', (data) => data.dosState >= 100, 0, { afterId: anchor }); + const app = await getAppContainerStatus(container, appName); + expect(app && app.status.startsWith('Up'), 'the app is still up').to.equal(true); + }); + }); + + describe('a stall past the threshold', function () { + let anchor; + + before(async function () { + anchor = client.getLastEventId(); + step('freezing'); + await freezeDocker(container); + }); + + after(async function () { + await thawEveryNode([client]); + }); + + it('held docker in the kernel', async function () { + const stuck = await stuckThreads(container, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + }); + + it('takes the node out of service, naming the stall', async function () { + this.timeout(STALL_MS + READ_INTERVAL_MS * 6); + await client.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }); + step('out of service'); + }); + + it('leaves nothing of the app that can run, though its removal could not finish', async function () { + this.timeout(REMOVAL_DEADLINE_MS * 4); + const { data } = await client.waitForEvent('hostStall:appUnloaded', (d) => d.app === appName, REMOVAL_DEADLINE_MS * 3, { afterId: anchor }); + expect(data.removal, 'the app\'s removal finished, so its stall was never what removed it').to.not.equal('removed'); + expect(data.runnable, 'the node reports a process of the app that can still run').to.deep.equal([]); + expect(await sliceProcessesThatCanRun(container, appName), 'a process of the app can still run').to.deep.equal([]); + }); + + it('stays out of service through a FluxOS restart in the same boot', async function () { + this.timeout(300000); + const restartAnchor = client.getLastEventId(); + step('restarting FluxOS'); + await restartFluxos(container, { timeout: 120000, readyTimeoutMs: 180000 }); + await client.waitForEvent('dos:changed', outOfServiceForTheStall, 60000, { afterId: restartAnchor }); + }); + + it('stays out of service once docker runs again', async function () { + this.timeout(READ_INTERVAL_MS * 6); + const thawAnchor = client.getLastEventId(); + step('thawing'); + await thawDocker(container); + await assertNoEvent(client, 'dos:changed', (data) => data.dosState < 100, READ_INTERVAL_MS * 3, { afterId: thawAnchor }); + const { data } = await client.getDOSState(); + expect(data.dosState, 'the node returned to service without a reboot').to.be.at.least(100); + }); + }); + + describe('a stall past the threshold, with docker unable to act at all, on a legacy node whose FluxOS is not root', function () { + const killedName = `e2ehoststallkill${Date.now()}`; + let killClient; + let killContainer; + let anchor; + let runnableAtTrip; + // the app's container's own cgroup, wherever this tree puts it + let appCgroup; + + before(async function () { + this.timeout(240000); + const index = await installOn(env, killedName, [LEGACY_UNPRIVILEGED]); + killClient = env.clients[index]; + ({ container: killContainer } = killClient); + await waitForUp(killClient, killedName, 'the app is up'); + appCgroup = await containerCgroup(killContainer, containerOf(killedName)); + anchor = killClient.getLastEventId(); + step(`freezing node ${index}`); + await freezeDocker(killContainer); + }); + + after(async function () { + if (killClient) await thawEveryNode([killClient]); + }); + + it('runs FluxOS as an account that is not root', async function () { + const owner = await execInContainer(killContainer, 'stat -c %U /proc/$(systemctl show -p MainPID --value fluxos)'); + expect(owner.stdout.trim(), 'FluxOS runs as root, so nothing below tested an operator\'s own install').to.equal('fluxuser'); + }); + + it('held docker in the kernel', async function () { + const stuck = await stuckThreads(killContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + }); + + it('left docker unable to answer', async function () { + this.timeout(30000); + await freezeUnit(killContainer, DOCKERD_UNIT); + const answer = await execInContainer(killContainer, 'timeout 5 docker version'); + expect(answer.exitCode, 'docker still answers, so it could still stop the app itself').to.not.equal(0); + const stuck = await stuckThreads(killContainer, 'dockerd'); + expect(stuck, 'docker\'s held thread was released by its freeze, so the detector has nothing to read').to.not.be.empty; + expect(await processesThatCanRun(killContainer, appCgroup), 'the app is not running, so there is nothing to kill').to.not.be.empty; + }); + + it('takes the node out of service', async function () { + this.timeout(STALL_MS + READ_INTERVAL_MS * 6); + await killClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }); + // read before the removal's deadline, while only a kill can end the app + runnableAtTrip = await processesThatCanRun(killContainer, appCgroup); + }); + + it('kills the app its removal could not stop, and nothing of it can run', async function () { + // every app on the node is removed in turn, each within its deadline + this.timeout(REMOVAL_DEADLINE_MS * 7); + expect(runnableAtTrip, 'nothing of the app could run when the node went out of service, so there was nothing to kill').to.not.be.empty; + const { data } = await killClient.waitForEvent('hostStall:appUnloaded', (d) => d.app === killedName, REMOVAL_DEADLINE_MS * 6, { afterId: anchor }); + expect(data.removal, 'the app\'s removal finished, so docker could act').to.not.equal('removed'); + expect(data.killed, 'the node never killed the app').to.equal(true); + expect(data.runnable, 'the node reports a process of the app that can still run').to.deep.equal([]); + expect(await sliceProcessesThatCanRun(killContainer, killedName), 'a process of the app can still run').to.deep.equal([]); + }); + }); + + describe('a stall past the threshold, with docker unable to act at all, on a node holding several apps', function () { + const stamp = Date.now(); + const names = [1, 2, 3].map((n) => `e2ehoststallmany${n}${stamp}`); + let manyClient; + let manyContainer; + let anchor; + let runningBefore; + let trippedAt; + let stopped; + let runnableAfterWindow; + + before(async function () { + this.timeout(480000); + const index = ARCANE.find((i) => i !== node); + manyClient = env.clients[index]; + ({ container: manyContainer } = manyClient); + // eslint-disable-next-line no-restricted-syntax + for (const name of names) { + // eslint-disable-next-line no-await-in-loop + await installOn(env, name, [index]); + // eslint-disable-next-line no-await-in-loop + await waitForUp(manyClient, name, `${name} is up`); + } + anchor = manyClient.getLastEventId(); + step(`freezing node ${index}, which holds ${names.length} apps`); + await freezeDocker(manyContainer); + await freezeUnit(manyContainer, DOCKERD_UNIT); + runningBefore = await Promise.all(names.map((name) => sliceProcessesThatCanRun(manyContainer, name))); + const trip = await manyClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + stopped = (await manyClient.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + // read once the window and the kill are over, and before even one removal's deadline + runnableAfterWindow = trippedAt + ? await Promise.all(names.map((name) => sliceProcessesThatCanRun(manyContainer, name))) + : null; + step('read every app after the stop window'); + }); + + after(async function () { + if (manyClient) await thawEveryNode([manyClient]); + }); + + it('held docker in the kernel, unable to answer, while every app could run', async function () { + const stuck = await stuckThreads(manyContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the freeze, so nothing below was tested').to.not.be.empty; + const answer = await execInContainer(manyContainer, 'timeout 5 docker version'); + expect(answer.exitCode, 'docker still answers, so it could still stop the apps itself').to.not.equal(0); + runningBefore.forEach((running, i) => expect(running, `${names[i]} was not running, so there was nothing to stop`).to.not.be.empty); + }); + + it('takes the node out of service', function () { + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('stops every app within one stop window, before even one removal\'s deadline', function () { + expect(STOP_WINDOW_MS + KILL_SLACK_MS, 'fixture: the window and the kill do not fit inside one removal\'s deadline').to.be.below(REMOVAL_DEADLINE_MS); + runnableAfterWindow.forEach((runnable, i) => expect(runnable, `${names[i]} can still run one stop window after the node went out of service`).to.deep.equal([])); + }); + + it('says every app was stopped, and that nothing of any can run', function () { + expect(stopped, 'the node never said its apps were stopped').to.not.equal(null); + expect(stopped.apps.map((a) => a.app), 'the apps the node stopped').to.include.members(names); + stopped.apps.filter((a) => names.includes(a.app)) + .forEach((a) => expect(a.runnable, `the node reports a process of ${a.app} that can still run`).to.deep.equal([])); + }); + }); + + describe('a stall past the threshold, on a node whose FluxOS restarts before the node goes out of service', function () { + const restartName = `e2ehoststallrestart${Date.now()}`; + let restartClient; + let restartContainer; + let anchor; + let inServiceAtRestart; + let restartedAt; + let trippedAt; + + before(async function () { + this.timeout(600000); + restartClient = env.clients[RESTARTED]; + ({ container: restartContainer } = restartClient); + await installOn(env, restartName, [RESTARTED]); + await waitForUp(restartClient, restartName, `${restartName} is up`); + anchor = restartClient.getLastEventId(); + step(`freezing node ${RESTARTED}'s docker`); + await freezeDocker(restartContainer); + // the second reading finds docker held; three readings after it, the node is + // two readings short of going out of service + await restartClient.waitForEvent('placementHold:set', byOwner, RESTART_READ_INTERVAL_MS * 3, { afterId: anchor }); + await new Promise((resolve) => { setTimeout(resolve, RESTART_READ_INTERVAL_MS * 3.25); }); + inServiceAtRestart = (await restartClient.getDOSState()).data.dosState < 100; + const restartAnchor = restartClient.getLastEventId(); + restartedAt = Date.now(); + step('restarting FluxOS'); + await restartFluxos(restartContainer, { timeout: 120000, readyTimeoutMs: 180000 }); + const trip = await restartClient.waitForEvent('dos:changed', outOfServiceForTheStall, RESTART_READ_INTERVAL_MS * (STUCK_INTERVALS + 3), { afterId: restartAnchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + step('out of service after the restart'); + }); + + after(async function () { + if (restartClient) await thawEveryNode([restartClient]); + }); + + it('held docker in the kernel across the restart, the node still in service when FluxOS restarted', async function () { + const stuck = await stuckThreads(restartContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the restart, so nothing below was tested').to.not.be.empty; + expect(inServiceAtRestart, 'fixture: the node was out of service before FluxOS restarted').to.equal(true); + }); + + it('goes out of service after the restart when it would have without it', function () { + expect(trippedAt, 'the node never went out of service after the restart').to.not.equal(null); + // two readings were left when FluxOS restarted; a count started again needs all of them + expect(trippedAt - restartedAt, 'the restart started the count of readings again').to.be.below(RESTART_READ_INTERVAL_MS * 3); + }); + }); + + describe('a stall on a node whose FluxOS restarts over and over', function () { + const loopName = `e2ehoststallloop${Date.now()}`; + let loopClient; + let loopContainer; + let restartsTookMs; + + before(async function () { + this.timeout(900000); + loopClient = env.clients[CRASH_LOOPED]; + ({ container: loopContainer } = loopClient); + await installOn(env, loopName, [CRASH_LOOPED]); + await waitForUp(loopClient, loopName, `${loopName} is up`); + step(`freezing node ${CRASH_LOOPED}'s docker`); + await freezeDocker(loopContainer); + const from = Date.now(); + for (let i = 0; i < CRASH_LOOP_RESTARTS; i += 1) { + step(`restarting FluxOS, ${i + 1} of ${CRASH_LOOP_RESTARTS}`); + // eslint-disable-next-line no-await-in-loop + await restartFluxos(loopContainer, { timeout: 120000, readyTimeoutMs: 180000 }); + } + restartsTookMs = Date.now() - from; + }); + + after(async function () { + if (loopClient) await thawEveryNode([loopClient]); + }); + + it('held docker in the kernel across the restarts, every one of them inside two readings', async function () { + const stuck = await stuckThreads(loopContainer, 'dockerd'); + expect(stuck, 'no dockerd thread was held across the restarts, so nothing below was tested').to.not.be.empty; + expect(restartsTookMs, 'fixture: the restarts took long enough for the readings alone to count').to.be.below(CRASH_LOOP_READ_INTERVAL_MS * 2); + }); + + // Out of service for a stall lasts through every restart, so the state after the + // last one says whether any of them took the node out. + it('stays in service, the restarts adding nothing to the count', async function () { + expect((await loopClient.getDOSState()).data.dosState, 'the restarts took the node out of service').to.be.below(100); + }); + }); + + describe('a stall past the threshold, on a node whose docker disk stops answering reads', function () { + const diskName = `e2ehoststalldisk${Date.now()}`; + let diskClient; + let diskContainer; + let anchor; + let readsWereHeld; + let runningBefore; + let trippedAt; + let stopped; + let runnableAfterWindow; + + before(async function () { + this.timeout(480000); + diskClient = env.clients[DISK_STALLED]; + ({ container: diskContainer } = diskClient); + await installOn(env, diskName, [DISK_STALLED]); + await waitForUp(diskClient, diskName, `${diskName} is up`); + anchor = diskClient.getLastEventId(); + step(`stalling node ${DISK_STALLED}'s docker disk`); + await stallDisk(diskContainer, DOCKER_ROOT); + readsWereHeld = await readsHeld(diskContainer, `${DOCKER_ROOT}/containers`, { file: 'config.v2.json' }); + runningBefore = await sliceProcessesThatCanRun(diskContainer, diskName); + const trip = await diskClient.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + // the records' read waits one window, and the stop another + stopped = (await diskClient.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS * 2 + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + runnableAfterWindow = trippedAt ? await sliceProcessesThatCanRun(diskContainer, diskName) : null; + step('read the app after the stop window'); + }); + + after(async function () { + if (!diskClient) return; + await resumeDisk(diskContainer, DOCKER_ROOT).catch(() => {}); + await thawEveryNode([diskClient]); + }); + + it('held every read of docker\'s records of its containers, while the app could run', function () { + expect(readsWereHeld, 'a read of docker\'s container records returned, so the disk was never held').to.equal(true); + expect(runningBefore, 'the app was not running, so there was nothing to stop').to.not.be.empty; + }); + + it('takes the node out of service', function () { + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('stops the app though docker\'s records cannot be read', function () { + expect(STOP_WINDOW_MS * 2 + KILL_SLACK_MS, 'fixture: the record read, the window and the kill do not fit inside one removal\'s deadline').to.be.below(REMOVAL_DEADLINE_MS); + expect(stopped, 'the node never stopped its apps while docker\'s records could not be read').to.not.equal(null); + expect(runnableAfterWindow, `${diskName} can still run after the stop window`).to.deep.equal([]); + }); + + it('says nothing of the app can run, every container on the node being in its slice', function () { + const app = stopped?.apps.find((a) => a.app === diskName); + expect(app, 'the node did not name the app among those it stopped').to.not.equal(undefined); + expect(app.runnable, 'the node reports the app as one it cannot tell can still run, or names its processes').to.deep.equal([]); + }); + }); +}); diff --git a/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js new file mode 100644 index 0000000000..c9e5bd0ff5 --- /dev/null +++ b/test-infra/runner/tests/3304-a-sync-answer-the-node-cannot-record-is-not-counted.js @@ -0,0 +1,131 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { + waitForDaemonReady, waitForNodeStatus, waitFor, +} from '../framework/wait.js'; +import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; +import { failCommands, failWrites } from '../framework/mongo-control.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node joining the network asks a peer for the network's app state, and the peer +// answers in several messages, only the last saying it is done. Here the joining +// node's database fails a call made while it records the first message - a write +// that reports it stored part of the batch, or the prune of old location rows, +// which throws - and the node must not count that peer as having answered: it ends +// the request and asks another peer. +// +// The harness keeps one sync request open at a time (appSyncMinCompletions), so a +// peer counted despite the failed write leaves one peer asked, and a peer replaced +// leaves two. syncAnswerBatchSize 2 splits an answer of a few running-app events +// into several messages, and a long locationTtlS keeps the fleet from announcing +// again while the node syncs, so the only running-app writes it makes then are the +// answer's. + +const ANSWER_BATCH = 2; +const NODES = 6; +const JOINING = NODES - 1; +// what the peers hold when the node joins: more events than one message carries +const ANSWERED_EVENTS = ANSWER_BATCH + 1; +// a peer's request ended for any of these reasons is replaced too, and is not this +const OTHER_ENDINGS = ['ephemeralSync:peerTimedOut', 'ephemeralSync:peerDisconnected', 'ephemeralSync:peerUnverified']; + +function step(what) { + console.log(`# 3304 ${new Date().toISOString()} ${what}`); +} + +const FAILURES = [ + { + what: 'a location write the database fails', + arm: (nodeNum) => failWrites(nodeNum, 'globalzelapps', 'zelappslocation', { times: 1 }), + }, + { + what: 'a location prune the database fails, which throws', + arm: (nodeNum) => failCommands(nodeNum, 'globalzelapps', 'zelappslocation', { commands: ['delete'], times: 1 }), + }, +]; + +FAILURES.forEach(({ what, arm }) => describe(`a sync answer the node cannot record is not counted: ${what}`, function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2esyncwrite${Date.now()}`; + let release; + let joining; + let eventsHeld; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + deferredNodes: 1, + tickerAutostart: false, + configOverrides: { + fluxapps: { + syncAnswerBatchSize: ANSWER_BATCH, + locationTtlS: 1800, + // the production budget: the per-peer deadlines derive from it, and at + // the harness's 30s the wait between two messages of an answer cut this + // small is shorter than the peer takes to send them + syncTimeoutMs: 120000, + }, + }, + }); + await bootAndPeer(env); + await seedSimpleApp(env, appName); + await waitFor(async () => (await dbClient(1).appStateEvents({ type: 'apprunning' })).length >= ANSWERED_EVENTS, { + timeout: 180000, interval: 2000, label: 'the peers hold more running-app events than one message carries', + }); + eventsHeld = (await dbClient(1).appStateEvents({ type: 'apprunning' })).length; + // Every peer answers before the node joins: a peer still syncing refuses it, + // and asking a third peer then says nothing about the one replaced. + await Promise.all(env.clients.slice(0, JOINING).map((client, i) => waitFor( + () => client.getEventBuffer().some((e) => e.event === 'orchestrator:stateChanged' && e.data?.to === 'READY'), + { timeout: 300000, interval: 1000, label: `node ${i} answers a state sync` }, + ))); + + step(`arming ${what}`); + release = await arm(JOINING + 1); + joining = await env.startNode(JOINING); + await waitForDaemonReady(joining); + await waitForNodeStatus(joining, (d) => d.confirmed === true, 60000); + await joining.waitForEvent('ephemeralSync:allComplete', () => true, 180000); + step('synced'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('was answered in more than one message', function () { + expect(eventsHeld, 'the answer fits one message, so no early message could fail').to.be.above(ANSWER_BATCH); + }); + + it('had the call refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database failed none of the joining node\'s calls, so nothing below was tested').to.equal(1); + }); + + it('asks another peer in place of the one whose answer it could not record', function () { + const asked = new Set(joining.getEventBuffer() + .filter((e) => e.event === 'ephemeralSync:requested') + .flatMap((e) => e.data.peers)); + const otherEndings = joining.getEventBuffer().filter((e) => OTHER_ENDINGS.includes(e.event)); + expect(otherEndings, 'a request ended for another reason, so a second peer proves nothing').to.deep.equal([]); + expect(asked.size, 'the peer whose answer was not recorded was counted, and nobody else was asked').to.equal(2); + }); + + it('names that peer, and never counts its running-app answer', function () { + const events = joining.getEventBuffer(); + const failedAt = events.find((e) => e.event === 'ephemeralSync:peerWriteFailed'); + expect(failedAt, 'the node said nothing about the answer it could not record').to.not.equal(undefined); + // the streams it finished before the failed write stand; the one it could not record does not + const counted = events.filter((e) => e.event === 'ephemeralSync:peerComplete' + && e.data.syncType === 'apprunning' && e.data.peer === failedAt.data.peer); + expect(counted, 'the peer whose answer was not recorded was counted').to.deep.equal([]); + }); +})); diff --git a/test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js b/test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js new file mode 100644 index 0000000000..cd3f929553 --- /dev/null +++ b/test-infra/runner/tests/3305-a-running-node-applies-late-messages-app-by-app.js @@ -0,0 +1,146 @@ +/* + * A node that is running finds more than 500 app messages missing behind payments its scan + * recorded, and its hash retry fetches them in bulk. It writes the registry row of each app + * those messages belong to, and touches no other: the registry is built in full only on the + * way to ready. + * + * The node boots holding two apps. The late log then reaches the peers, and the node's + * payment records for it are written as its scan of the chain would have written them: + * ten new apps with long update chains, and late updates to one of the two it already holds. + * The other is untouched by the late log, so its row is only rewritten by a full rebuild, + * which replaces every row of the registry. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { waitForDaemonReady, waitForOrchestratorState, waitFor } from '../framework/wait.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const NODES = 4; +// the running node: held back until the apps it boots holding are in place +const SUBJECT = NODES - 1; +const LATE_APPS = 10; +const UPDATES = 54; +const LATE_UPDATES_TO_HELD = 5; + +const unresolved = (entry) => ({ ...entry, message: false }); + +function step(what) { + console.log(`# 3305 ${new Date().toISOString()} ${what}`); +} + +describe('a running node applies late messages app by app', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + // the newest message of each app the late log brings, by name + const governing = new Map(); + let untouched; + let updated; + let rowsBefore; + let lateCount; + let bulkFetched; + const db = () => dbClient(SUBJECT + 1); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + deferredNodes: 1, + // the retry that finds the late log comes within a few blocks + configOverrides: { fluxapps: { hashSyncFallbackRecheckBlocks: 2 } }, + }); + await bootAndPeer(env); + const base = env.initialHeight - 2000; + + // the two apps the node boots holding, its rows for them written by its own rebuild + untouched = await buildSeedableApp({ env, name: `e2eHeld${stamp}`, height: base }); + updated = await buildSeedableApp({ env, name: `e2eHeldUp${stamp}`, height: base + 1 }); + const held = [untouched, updated]; + await Promise.all(Array.from({ length: NODES }, (_, i) => i).map(async (i) => { + await dbClient(i + 1).seedPermanentMessages(held.map((m) => m.permanentMessage)); + await dbClient(i + 1).seedAppHashes(held.map((m) => m.hashEntry)); + })); + const subject = await env.startNode(SUBJECT); + await waitForDaemonReady(subject); + await env.startDiscovery([SUBJECT]); + await waitForOrchestratorState(subject, 'READY', 600000); + await waitFor(async () => (await db().globalAppSpec(updated.permanentMessage.appSpecifications.name))?.hash === updated.hash + && (await db().globalAppSpec(untouched.permanentMessage.appSpecifications.name))?.hash === untouched.hash, { + timeout: 120000, interval: 2000, label: 'the node holds the two apps it booted with', + }); + rowsBefore = { + untouched: await db().globalAppSpec(untouched.permanentMessage.appSpecifications.name), + updated: await db().globalAppSpec(updated.permanentMessage.appSpecifications.name), + }; + + // the late log + const late = []; + let current = updated; + for (let j = 1; j <= LATE_UPDATES_TO_HELD; j += 1) { + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableUpdate(current, (spec) => { spec.description = `held app revision ${j}`; }, { height: base + 2 + j }); + late.push(current); + } + governing.set(updated.permanentMessage.appSpecifications.name, current.hash); + for (let k = 0; k < LATE_APPS; k += 1) { + const name = `e2eLate${k}x${stamp}`; + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableApp({ env, name, height: base + 100 + (k * 60) }); + late.push(current); + for (let j = 1; j <= UPDATES; j += 1) { + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableUpdate(current, (spec) => { spec.description = `${name} revision ${j}`; }, { height: base + 100 + (k * 60) + j }); + late.push(current); + } + governing.set(name, current.hash); + } + lateCount = late.length; + expect(lateCount, 'enough messages late to take the bulk path').to.be.above(500); + + const anchor = subject.getLastEventId(); + step(`seeding ${lateCount} late messages`); + await Promise.all(env.clients.map(async (client, i) => { + if (!client || i === SUBJECT) return; + await dbClient(i + 1).seedPermanentMessages(late.map((m) => m.permanentMessage)); + await dbClient(i + 1).seedAppHashes(late.map((m) => m.hashEntry)); + })); + await db().seedAppHashes(late.map((m) => unresolved(m.hashEntry))); + bulkFetched = await subject.waitForEvent('hashSync:bulkFetched', (d) => d.processed > 0, 600000, { afterId: anchor }); + step('fetched in bulk'); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('took the late messages in bulk while running', function () { + expect(bulkFetched.data.processed, JSON.stringify(bulkFetched.data)).to.equal(lateCount); + }); + + it('holds every app the late log brings at its newest message', async function () { + this.timeout(180000); + const names = [...governing.keys()]; + await waitFor(async () => { + const held = await Promise.all(names.map(async (name) => (await db().globalAppSpec(name))?.hash)); + return held.every((hash, i) => hash === governing.get(names[i])); + }, { timeout: 150000, interval: 3000, label: 'every late app held at its newest message' }); + }); + + it('brings an app it already held up to its late update, in the same row', async function () { + const row = await db().globalAppSpec(updated.permanentMessage.appSpecifications.name); + expect(row.hash).to.equal(governing.get(updated.permanentMessage.appSpecifications.name)); + expect(String(row._id), 'the row was replaced, so the registry was rebuilt').to.equal(String(rowsBefore.updated._id)); + }); + + it('rewrites no row the late log does not touch', async function () { + const row = await db().globalAppSpec(untouched.permanentMessage.appSpecifications.name); + expect(row, 'the app the late log does not touch is gone').to.not.equal(null); + expect(String(row._id), 'the row was replaced, so the registry was rebuilt').to.equal(String(rowsBefore.untouched._id)); + }); +}); diff --git a/test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js b/test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js new file mode 100644 index 0000000000..88910c359d --- /dev/null +++ b/test-infra/runner/tests/3306-a-redeploy-whose-record-write-fails-keeps-the-app.js @@ -0,0 +1,120 @@ +// weight: heavy +/* + * The periodic reinstall pass redeploys an app whose specification changed, and the + * database refuses the write of the app's new record. The app keeps its record, at the + * old specification, so the node still knows it holds the app: the reconciler restores + * what the pass took down, and a later pass redeploys it at the new specification. + * + * The change arrives as in suite 85: the app's global row is replaced and a new + * permanent message and hash seeded beside it. The pass fires on + * `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0`, every 16 to 36 + * blocks here, so the drive is budgeted in blocks. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { getAppContainerId } from '../framework/container.js'; +import { waitFor, waitForComponentRedeployed } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { failWrites } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// four passes at the longest period: the one whose write is refused, and others after it +const PASS_BLOCKS = 144; +const HOLDER = 0; + +describe('a redeploy whose record write fails keeps the app', function () { + let env; + dumpLogsOnFailure(() => env); + + const appName = `e2eredeployrow${Date.now()}`; + let holder; + let app; + let updated; + let containerBefore; + let release; + let beforeUpdate; + + async function localSpec(client) { + const res = await client.getInstalledApps(); + if (res.status !== 'success') return null; + return (res.data ?? []).find((a) => a.name === appName) ?? null; + } + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ hookCtx: this, nodes: 5, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await pushImage(appName, 'v1'); + app = await buildSeedableApp({ + env, + name: appName, + instances: 1, + compose: [{ + name: appName, + description: 'redeploy row test container', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + await installOnNodes(env, app, [HOLDER]); + await waitFor(async () => { + containerBefore = await getAppContainerId(holder.container, appName, appName); + return Boolean(containerBefore); + }, { timeout: 120000, interval: 2000, label: 'the app is running before its spec changes' }); + + // a description change takes the soft path, which keeps the data volume + updated = await buildSeedableUpdate(app, (spec) => { + spec.compose[0].description = 'redeploy row test container, second specification'; + }); + beforeUpdate = holder.getLastEventId(); + await seedSpecUpdate(env, updated, [HOLDER]); + // the only thing advancing the chain from here is the drive below + await stopTicker(); + release = await failWrites(HOLDER + 1, 'localzelapps', 'zelappsinformation', { times: 1 }); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('comes back at the new specification', async function () { + // a backstop: the drive is bounded in blocks, and blocks cost what the node takes + this.timeout(2400000); + const replaced = () => waitForComponentRedeployed(holder, appName, appName, false, 0, { afterId: beforeUpdate }) + .then(() => true, () => false); + await driveUntil(holder, replaced, { + blocks: PASS_BLOCKS, label: 'the node redeploys the app at the new specification', + }); + + const installed = await localSpec(holder); + expect(installed, 'the node no longer holds the app').to.not.equal(null); + expect(installed.hash, 'the row is at the new specification').to.equal(updated.hash); + const containerAfter = await getAppContainerId(holder.container, appName, appName); + expect(containerAfter, 'the app is not running').to.be.a('string'); + expect(containerAfter, 'the component was never replaced').to.not.equal(containerBefore); + }); + + it('had the write of the app\'s record refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database refused none of the node\'s record writes, so nothing above was tested').to.equal(1); + }); +}); diff --git a/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js b/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js new file mode 100644 index 0000000000..dec8d44e94 --- /dev/null +++ b/test-infra/runner/tests/3307-an-app-upgraded-from-the-old-format-is-rebuilt.js @@ -0,0 +1,170 @@ +// weight: heavy +/* + * An app installed at specification version 3 is updated to a version that has + * components. Nothing of the old app carries over, so the reinstall pass rebuilds it + * from nothing: removed, and installed again at the new specification. + * + * On one node the rebuild completes, and the app runs at its new specification. On + * another the database refuses the write of the new app's row, so the node no longer + * holds the app and tells the network so; the pass goes on to the next app. + * + * The change arrives as in suite 85. The pass fires on + * `blockHeight % (updateFluxAppsPeriod * speedMultiplier) === 0`, every 16 to 36 + * blocks here, so the drive is budgeted in blocks. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp, buildSeedableLegacyApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes, seedSpecUpdate } from '../framework/reconciler-suite.js'; +import { execInContainer, getAppContainerId } from '../framework/container.js'; +import { waitFor, waitForComponentRedeployed } from '../framework/wait.js'; +import { driveUntil, stopTicker } from '../framework/daemon-control.js'; +import { failWrites } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const PASS_BLOCKS = 144; +const FAILS = 0; +const UPGRADES = 1; + +function step(what) { + console.log(`# 3307 ${new Date().toISOString()} ${what}`); +} + +const component = (name, description) => ({ + name, + description, + repotag: `${REGISTRY_REPO_HOST}/${name}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', +}); + +// The specification an old-format app is updated to: the one a new app of its name +// would carry, signed by its owner as an update. +async function upgradeOf(env, legacy) { + const { name } = legacy.spec; + const target = await buildSeedableApp({ env, name, instances: 1, compose: [component(name, `${name} upgraded`)] }); + return buildSeedableUpdate(legacy, (spec) => { + Object.keys(spec).forEach((key) => { delete spec[key]; }); + const { hash, height, ...fields } = target.spec; + Object.assign(spec, fields); + }); +} + +// an app at specification version 3 has one container, named for the app alone +async function legacyContainerId(container, name) { + const { stdout } = await execInContainer(container, `docker inspect --format '{{.Id}}' flux${name} 2>/dev/null || echo ""`); + return stdout.trim() || null; +} + +async function localSpec(client, name) { + const res = await client.getInstalledApps(); + if (res.status !== 'success') return null; + return (res.data ?? []).find((a) => a.name === name) ?? null; +} + +describe('an app upgraded from the old format is rebuilt', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + const failing = `e2elegacyfail${stamp}`; + const other = `e2elegacynext${stamp}`; + const upgrading = `e2elegacyup${stamp}`; + let upgraded; + let release; + let anchors; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 5, tickerAutostart: false }); + await bootAndPeer(env); + + await Promise.all([failing, upgrading, other].map((name) => pushImage(name, 'v1'))); + const legacyFailing = await buildSeedableLegacyApp({ env, name: failing }); + const legacyUpgrading = await buildSeedableLegacyApp({ env, name: upgrading }); + const otherApp = await buildSeedableApp({ env, name: other, instances: 1, compose: [component(other, `${other} first`)] }); + // in this order: the reinstall pass reaches the app it cannot rebuild first + await installOnNodes(env, legacyFailing, [FAILS]); + await installOnNodes(env, otherApp, [FAILS]); + await installOnNodes(env, legacyUpgrading, [UPGRADES]); + await waitFor(async () => Boolean(await legacyContainerId(env.clients[FAILS].container, failing)) + && Boolean(await getAppContainerId(env.clients[FAILS].container, other, other)) + && Boolean(await legacyContainerId(env.clients[UPGRADES].container, upgrading)), { + timeout: 180000, interval: 2000, label: 'every app runs before its specification changes', + }); + + const failingUpgrade = await upgradeOf(env, legacyFailing); + upgraded = await upgradeOf(env, legacyUpgrading); + const otherUpdate = await buildSeedableUpdate(otherApp, (spec) => { spec.compose[0].description = `${other} second`; }); + anchors = env.clients.map((client) => client?.getLastEventId()); + await seedSpecUpdate(env, failingUpgrade, [FAILS]); + await seedSpecUpdate(env, otherUpdate, [FAILS]); + await seedSpecUpdate(env, upgraded, [UPGRADES]); + await stopTicker(); + release = await failWrites(FAILS + 1, 'localzelapps', 'zelappsinformation', { times: 1 }); + step('the reinstall pass is due'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('goes on to the next app in the same pass after one it could not rebuild', async function () { + this.timeout(2400000); + const node = env.clients[FAILS]; + const since = () => node.getEventBuffer().filter((e) => e.id > anchors[FAILS]); + const failedRebuild = () => since().find((e) => e.event === 'app:rebuilt' && e.data.name === failing && e.data.outcome !== 'installed'); + const passEndAfter = (event) => since().find((e) => e.event === 'reinstall:passEnded' && e.id > event.id); + await driveUntil(node, async () => Boolean(failedRebuild() && passEndAfter(failedRebuild())), { + blocks: PASS_BLOCKS, label: 'a reinstall pass rebuilds the app, fails, and ends', + }); + const rebuilt = failedRebuild(); + const ended = passEndAfter(rebuilt); + const redeployed = since().find((e) => e.event === 'app:componentRedeployed' && e.data.name === other + && e.id > rebuilt.id && e.id < ended.id); + expect(redeployed, 'the pass stopped at the app it could not rebuild').to.not.equal(undefined); + }); + + it('tells the network it no longer holds the app it could not rebuild', async function () { + this.timeout(120000); + const node = env.clients[FAILS]; + const peers = env.clients.filter((client, i) => client && i !== FAILS); + const heard = await Promise.any(peers.map((peer, i) => peer.waitForEvent( + 'network:appremoved', + (d) => d.name === failing && d.ip.startsWith(node.ip), + 60000, + { afterId: anchors[env.clients.indexOf(peer)] }, + ).then(() => i))).catch(() => null); + expect(heard, 'no node was told the app had gone').to.not.equal(null); + }); + + it('had the write of the rebuilt app\'s row refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database refused none of the node\'s row writes, so nothing above was tested').to.equal(1); + }); + + it('rebuilds an app whose upgrade completes at its new specification', async function () { + this.timeout(2400000); + const node = env.clients[UPGRADES]; + const atNewSpec = async () => (await localSpec(node, upgrading))?.hash === upgraded.hash; + await driveUntil(node, atNewSpec, { blocks: PASS_BLOCKS, label: 'the node rebuilds the app at its new specification' }); + const installed = await localSpec(node, upgrading); + expect(installed.version, 'the app is not at its new specification version').to.equal(upgraded.spec.version); + await waitFor(async () => Boolean(await getAppContainerId(node.container, upgrading, upgrading)), { + timeout: 120000, interval: 2000, label: 'the rebuilt app runs as a component', + }); + }); +}); diff --git a/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js new file mode 100644 index 0000000000..637371ecfc --- /dev/null +++ b/test-infra/runner/tests/3308-boot-holds-an-app-it-cannot-confirm.js @@ -0,0 +1,502 @@ +// weight: heavy +/* + * At boot a node reads its location record of each app whose containers are stopped, + * which is every app a machine reboot leaves. One whose record has expired was placed + * elsewhere while the node was down and is removed; one whose record is valid is + * started. When the record cannot be read the node cannot tell, so the app is held: + * not started, not elected, not announced. Boot reads the record again until it + * answers, then starts or removes the app by the answer. + * + * Here FluxOS is killed and held down, the apps' containers are stopped, and the + * node's reads of the location collection are refused before FluxOS comes back. The + * holder's records are made to outlive the restart, as a production record does and + * the harness's compressed one would not. + * + * A record that has expired is one the holder last renewed longer ago than its TTL, + * everywhere: the holder stays down that long. A node down that long whose boot + * history says so removes every app at once; one with no boot history - a FluxOS + * that never wrote one - decides app by app, which is the path tested here. + * + * The mixed app pairs an active-standby component, which only the election starts, + * with a plain one boot starts. Its decision is held at a checkpoint once the + * database answers, so the announcement and the election that run then can be + * watched leaving it alone. The plain app is decided as soon as the database + * answers: it is the canary that the announcement and the decisions run at all. + * + * Every app asks for as many instances as the fleet has nodes and is installed on + * every node, the holder first so it is the mixed app's primary, before the holder + * goes down: none is placed anew while the holder is away. A standby may take over + * the mixed app's active-standby component in that time. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage, pushTestApp } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { + bootAndPeer, installOnNodes, installSeededOnNodes, installedInstanceIndices, +} from '../framework/reconciler-suite.js'; +import { + crashFluxos, listAppContainers, releaseFluxos, stopAppContainer, +} from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dbClient } from '../framework/db-client.js'; +import { isDaemonUp } from '../framework/syncthing-real.js'; +import { gComponent, watchGComponents } from '../framework/g-components.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; +import { loadSharedConfig, BOOT_DRIFT_MS } from '../framework/coupled-knobs.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { authenticate } from '../auth.js'; +import { fluxTeamKey } from '../framework/keys.js'; + +const HOLDER = 0; +const NODES = 3; +const APP_UID = 1000; +const BEFORE_DECISION = 'boot:beforeLocationDecision'; +// Longer than anything below, and far shorter than a production record's TTL. +const RECORD_OUTLIVES_MS = 60 * 60 * 1000; +// Down this long, every location record of the holder has expired on every node. +const RECORDS_EXPIRE_MS = loadSharedConfig().fluxapps.locationTtlS * 1000 + BOOT_DRIFT_MS; + +function step(what) { + console.log(`# 3308 ${new Date().toISOString()} ${what}`); +} + +function plainComponent(appName) { + return { + name: appName, + description: 'boot location test container', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +describe('boot holds an app whose location it cannot read', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + // decided at once when the database answers + const plainApp = `e2ebootplain${stamp}`; + // decided once the checkpoint is released + const mixedApp = `e2ebootmixed${stamp}`; + // its record is gone + const lapsedApp = `e2ebootlapsed${stamp}`; + // its record is gone, and it is uninstalled and installed again while its decision is held + const againApp = `e2ebootagain${stamp}`; + const db = `db_${mixedApp}`; + const web = `web_${mixedApp}`; + const plain = `${plainApp}_${plainApp}`; + const lapsed = `${lapsedApp}_${lapsedApp}`; + const again = `${againApp}_${againApp}`; + let againSpec; + let holder; + let holderDb; + const activeStandby = watchGComponents(() => env, { + appName: mixedApp, components: ['db'], holders: [...Array(NODES).keys()], + }); + + const isUp = async (identifier) => (await listAppContainers(holder.container)) + .some((c) => c.name === `flux${identifier}` && c.status?.startsWith('Up')); + const installedNames = () => holderDb.localAppNames(); + const heardRemoved = (appName, afterIds) => env.clients.some((client, i) => i !== HOLDER && client + && client.getEventBuffer().some((e) => e.id > afterIds[i] && e.event === 'network:appremoved' + && e.data?.name === appName && e.data?.ip?.startsWith(holder.ip))); + const heardRunning = (appName, afterIds) => env.clients.some((client, i) => i !== HOLDER && client + && client.getEventBuffer().some((e) => e.id > afterIds[i] && e.event === 'network:apprunning' + && e.data?.ip?.startsWith(holder.ip) && (e.data?.apps ?? []).some((a) => a.name === appName))); + // What the reconciler did with a component it was asked about after `afterId`: + // deferred it or started it, resolved once it has done one of the two; null when + // it did neither. + const reconcilerAnswer = async (identifier, afterId) => { + let answer = null; + await waitFor(async () => { + const started = holder.getEventBuffer().some((e) => e.id > afterId && e.event === 'reconciler:actuated' + && e.data?.identifier === identifier && e.data?.action === 'started'); + if (started) { + answer = 'started'; + return true; + } + if (await holder.getDecisionCount('reconciler:deferred', identifier, 'managedElsewhere') > 0) { + answer = 'deferred'; + return true; + } + return false; + }, { timeout: 120000, interval: 1000, label: `the reconciler answered for ${identifier}` }).catch(() => {}); + return answer; + }; + const cycles = async () => (await holder.getTestCounters())?.['masterSlave:cycles'] ?? 0; + + // Each step below records what it saw rather than throwing, so a node that does + // not hold the app answers at the assertion that names what it did instead. + const heard = (name, predicate, timeout, afterId) => holder.waitForEvent(name, predicate, timeout, { afterId }) + .catch(() => null); + + // FluxOS killed and held, the given containers stopped, the holder's records of + // `keep` made to outlive the restart, and its reads of the location collection + // refused from before FluxOS comes back. `expire` holds it down until every record + // of it has expired, its boot history forgotten. + async function rebootUnreadable(identifiers, { keep = [], expire = false }) { + await crashFluxos(holder.container, { hold: true }); + for (const identifier of identifiers) { + const [component, appName] = identifier.split('_'); + // eslint-disable-next-line no-await-in-loop + const stopped = await stopAppContainer(holder.container, appName, component); + if (stopped.exitCode !== 0) throw new Error(`could not stop ${identifier}: ${stopped.output}`); + } + for (const appName of keep) { + // eslint-disable-next-line no-await-in-loop + const kept = await holderDb.extendAppLocation(appName, holder.ip, RECORD_OUTLIVES_MS); + if (kept < 1) throw new Error(`the holder has no location record of ${appName} to keep`); + } + if (expire) { + await sleepUnlessInfraDead(RECORDS_EXPIRE_MS); + await holderDb.forgetHeartbeat(); + } + const anchors = env.clients.map((client) => client?.getLastEventId()); + const release = await failCommands(HOLDER + 1, 'globalzelapps', 'zelappslocation', { commands: ['find'] }); + await releaseFluxos(holder.container, { readyTimeoutMs: 180000 }); + return { anchors, release }; + } + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + syncthing: 'binary', + tickerAutostart: false, + configOverrides: { + fluxapps: { minOutgoing: 1, minIncoming: 1 }, + }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + holder = env.clients[HOLDER]; + holderDb = dbClient(HOLDER + 1); + await Promise.all(env.clients.map((client, i) => waitFor(() => isDaemonUp(client), { + timeout: 180000, interval: 3000, label: `syncthing daemon up on node ${i}`, + }))); + + await pushTestApp(mixedApp, 'v1', 'bootmixed', { user: APP_UID }); + await pushImage(plainApp, 'v1'); + await pushImage(lapsedApp, 'v1'); + await pushImage(againApp, 'v1'); + const apps = await Promise.all([ + buildSeedableApp({ + env, + name: mixedApp, + instances: NODES, + compose: [gComponent(mixedApp, 'db'), { ...gComponent(mixedApp, 'web'), containerData: '/appdata' }], + }), + buildSeedableApp({ env, name: plainApp, instances: NODES, compose: [plainComponent(plainApp)] }), + buildSeedableApp({ env, name: lapsedApp, instances: NODES, compose: [plainComponent(lapsedApp)] }), + buildSeedableApp({ env, name: againApp, instances: NODES, compose: [plainComponent(againApp)] }), + ]); + [, , , againSpec] = apps; + for (const app of apps) { + // eslint-disable-next-line no-await-in-loop + await installOnNodes(env, app, [HOLDER]); + } + await waitFor(async () => (await Promise.all([db, web, plain, lapsed, again].map(isUp))).every(Boolean), { + timeout: 300000, interval: 3000, label: 'every app runs on the holder, the mixed app\'s db as its primary', + }); + const peers = [...Array(NODES).keys()].filter((i) => i !== HOLDER); + for (const app of apps) { + // eslint-disable-next-line no-await-in-loop + await installOnNodes(env, app, peers); + } + await waitFor(async () => (await Promise.all([mixedApp, plainApp, lapsedApp, againApp].map((appName) => installedInstanceIndices(env, appName)))) + .every((holders) => holders.length === NODES), { + timeout: 300000, interval: 5000, label: 'every app is on every node', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + describe('a record that reads valid once it can be read', function () { + let anchors; + let release; + let failedReads; + let stoppedAtBoot; + let sweptAfterBoot; + let webWhileUnreadable; + let plainWhileUnreadable; + let upWhileUnreadable; + let installedWhileUnreadable; + let announcement; + let electionCycles; + let evaluatedWhileHeld; + let skippedWhileHeld; + let upWhileHeld; + let heldAtBoot; + let parkedAtDecision; + + before(async function () { + this.timeout(900000); + step('taking the node down, its containers stopped and its location reads refused'); + ({ anchors, release } = await rebootUnreadable([db, web, plain], { keep: [mixedApp, plainApp] })); + // Held before boot's first read, which fails: the decision it holds comes at + // least one retry interval after that. + const holdable = await holder.holdCheckpoint(BEFORE_DECISION, mixedApp).then(() => true, () => false); + const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + heldAtBoot = (await Promise.all([mixedApp, plainApp].map((appName) => heard( + 'boot:appUnconfirmed', (d) => d.name === appName, 60000, anchors[HOLDER], + )))).every(Boolean); + stoppedAtBoot = !(await Promise.all([db, web, plain].map(isUp))).some(Boolean); + + step('the location records unreadable: the reconciler is asked about every component'); + sweptAfterBoot = Boolean(await holder.waitForEvent('reconciler:swept', () => true, 120000, { afterId: settled.id }) + .catch(() => null)); + [webWhileUnreadable, plainWhileUnreadable] = await Promise.all([ + reconcilerAnswer(web, settled.id), reconcilerAnswer(plain, settled.id), + ]); + upWhileUnreadable = (await Promise.all([db, web, plain].map(isUp))).some(Boolean); + installedWhileUnreadable = await installedNames(); + + step('the database answers: the plain app is decided, the mixed app held at its decision'); + const answeredFrom = holder.getLastEventId(); + const evaluatedBefore = await holder.getDecisionCount('masterSlave:decision', db, 'evaluated'); + failedReads = await release(); + release = null; + parkedAtDecision = holdable && Boolean(await heard( + 'checkpoint:held', (d) => d.name === BEFORE_DECISION && d.key === mixedApp, 120000, answeredFrom, + )); + await heard('boot:appConfirmed', (d) => d.name === plainApp && d.valid === true, 120000, answeredFrom); + announcement = (await heard( + 'app:running', (d) => (d.apps ?? []).some((a) => a.name === plainApp), 120000, answeredFrom, + ))?.data ?? null; + const cyclesFrom = await cycles(); + await waitFor(async () => (await cycles()) >= cyclesFrom + 2, { + timeout: 180000, interval: 2000, label: 'the election ran two passes while the mixed app was held', + }).catch(() => {}); + electionCycles = (await cycles()) - cyclesFrom; + evaluatedWhileHeld = (await holder.getDecisionCount('masterSlave:decision', db, 'evaluated')) - evaluatedBefore; + skippedWhileHeld = await holder.getDecisionCount('masterSlave:decision', mixedApp, 'skippedUnconfirmed'); + upWhileHeld = (await Promise.all([db, web].map(isUp))).some(Boolean); + + step('the mixed app\'s decision released'); + if (holdable) await holder.releaseCheckpoint(BEFORE_DECISION, mixedApp); + await heard('boot:appConfirmed', (d) => d.name === mixedApp && d.valid === true, 60000, answeredFrom); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await holder?.releaseAllCheckpoints().catch(() => {}); + }); + + it('boots with the apps\' containers stopped', function () { + expect(stoppedAtBoot, 'a container was running when FluxOS came back, so boot had nothing to check').to.equal(true); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('keeps an app it cannot confirm installed', function () { + expect(installedWhileUnreadable, 'boot removed an app whose location it could not read').to.include.members([mixedApp, plainApp]); + expect(heldAtBoot, 'boot did not hold the apps whose location it could not read').to.equal(true); + }); + + it('holds the mixed app\'s decision at its checkpoint once the database answers', function () { + expect(parkedAtDecision, 'the mixed app\'s decision was not held, so the announcement and election below were not watched while it was undecided').to.equal(true); + }); + + it('starts no component of an app it cannot confirm, when the reconciler is asked to', function () { + expect(sweptAfterBoot, 'no reconciler sweep ran while the records were unreadable, so nothing below was tested').to.equal(true); + expect(webWhileUnreadable, 'the reconciler started a component of an app boot could not confirm').to.equal('deferred'); + expect(plainWhileUnreadable, 'the reconciler started an app boot could not confirm').to.equal('deferred'); + expect(upWhileUnreadable, 'a component of an app boot could not confirm is running').to.equal(false); + }); + + it('leaves an app it has not confirmed out of the announcement once the database answers', function () { + expect(announcement, 'the node announced nothing once the database answered, so nothing below was tested').to.not.equal(null); + expect(announcement.apps.map((a) => a.name), 'the announcement named an app boot had not confirmed').to.not.include(mixedApp); + }); + + it('leaves an app it has not confirmed out of the election', function () { + expect(electionCycles, 'the election ran no pass while the app was held, so nothing below was tested').to.be.at.least(2); + expect(evaluatedWhileHeld, 'the election decided an app boot had not confirmed').to.equal(0); + expect(skippedWhileHeld, 'the election did not leave the held app alone').to.be.at.least(1); + expect(upWhileHeld, 'a component of an app boot had not confirmed is running').to.equal(false); + }); + + it('starts the app once its record reads valid, its active-standby component on one node', async function () { + this.timeout(240000); + await waitFor(async () => (await Promise.all([web, plain].map(isUp))).every(Boolean), { + timeout: 200000, interval: 3000, label: 'the plain components run again', + }); + await waitFor(async () => Object.values(await activeStandby.oneHolderAtMost()).some((running) => running.includes('db')), { + timeout: 200000, interval: 3000, label: 'the mixed app\'s db runs on one node', + }); + expect(await installedNames()).to.include.members([mixedApp, plainApp]); + }); + + it('tells the network nothing of an app it kept', async function () { + this.timeout(120000); + await waitFor(() => heardRunning(plainApp, anchors), { + timeout: 100000, interval: 2000, label: 'a peer heard the holder announce once the database answered', + }); + expect(heardRemoved(mixedApp, anchors), 'a peer was told the holder gave up an app it kept').to.equal(false); + expect(heardRemoved(plainApp, anchors), 'a peer was told the holder gave up an app it kept').to.equal(false); + }); + }); + + describe('a record that reads expired once it can be read', function () { + let anchors; + let release; + let failedReads; + let sweptAfterBoot; + let lapsedWhileUnreadable; + let installedWhileUnreadable; + let removed; + let installedAfter; + let heldAtBoot; + + before(async function () { + this.timeout(900000); + step('taking the node down again, for longer than its records live, its boot history forgotten'); + ({ anchors, release } = await rebootUnreadable([lapsed], { expire: true })); + const settled = await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + heldAtBoot = Boolean(await heard('boot:appUnconfirmed', (d) => d.name === lapsedApp, 60000, anchors[HOLDER])); + sweptAfterBoot = Boolean(await holder.waitForEvent('reconciler:swept', () => true, 120000, { afterId: settled.id }) + .catch(() => null)); + lapsedWhileUnreadable = await reconcilerAnswer(lapsed, settled.id); + installedWhileUnreadable = await installedNames(); + + step('the database answers'); + const answeredFrom = holder.getLastEventId(); + failedReads = await release(); + release = null; + await heard('boot:appConfirmed', (d) => d.name === lapsedApp && d.valid === false, 120000, answeredFrom); + removed = Boolean(await heard('app:removed', (d) => d.name === lapsedApp, 120000, answeredFrom)); + installedAfter = await installedNames(); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('holds the app, installed and stopped, while its record cannot be read', function () { + expect(sweptAfterBoot, 'no reconciler sweep ran while the records were unreadable, so nothing below was tested').to.equal(true); + expect(installedWhileUnreadable, 'boot removed an app whose location it could not read').to.include(lapsedApp); + expect(heldAtBoot, 'boot did not hold the app whose location it could not read').to.equal(true); + expect(lapsedWhileUnreadable, 'the reconciler started an app boot could not confirm').to.equal('deferred'); + }); + + it('removes the app once its record reads expired', function () { + expect(removed, 'the app whose record expired was not removed').to.equal(true); + expect(installedAfter, 'the app whose record expired is still installed').to.not.include(lapsedApp); + }); + + it('tells the network nothing of the removal, the network having dropped the record', async function () { + this.timeout(120000); + // the plain app runs throughout, so boot does not check it and the holder names it + await waitFor(() => heardRunning(plainApp, anchors), { + timeout: 100000, interval: 2000, label: 'a peer heard the holder announce once the database answered', + }); + expect(heardRemoved(lapsedApp, anchors), 'a peer was told of the removal of an app whose record had expired').to.equal(false); + }); + }); + + // The decision is held after boot has read the expired record. While it is held + // the app is uninstalled and installed again: the install is a new app, which the + // decision about the old one must leave alone. + describe('an app uninstalled and installed again while its decision is held', function () { + let release; + let failedReads; + let heldAtBoot; + let parkedAtDecision; + let installedAfterUninstall; + let installedAgain; + let decision; + let removedAfterDecision; + + before(async function () { + this.timeout(900000); + step('taking the node down for longer than its records live, its location reads refused'); + let anchors; + ({ anchors, release } = await rebootUnreadable([again], { expire: true })); + const holdable = await holder.holdCheckpoint(BEFORE_DECISION, againApp).then(() => true, () => false); + await holder.waitForEvent('boot:settled', () => true, 300000, { afterId: anchors[HOLDER] }); + heldAtBoot = Boolean(await heard('boot:appUnconfirmed', (d) => d.name === againApp, 60000, anchors[HOLDER])); + + step('the database answers: the expired record is read, its decision held'); + const answeredFrom = holder.getLastEventId(); + failedReads = await release(); + release = null; + parkedAtDecision = holdable && Boolean(await heard( + 'checkpoint:held', (d) => d.name === BEFORE_DECISION && d.key === againApp, 120000, answeredFrom, + )); + + step('uninstalling the app and installing it again while its decision is held'); + const auth = await authenticate(holder.url, fluxTeamKey()); + // Asked again until it is gone: the node refuses a removal while another runs, + // and boot is removing the apps whose records expired with this one's. + await waitFor(async () => { + if (!(await installedNames()).includes(againApp)) return true; + await (await fetch(`${holder.url}/apps/appremove/${againApp}`, { headers: { zelidauth: auth.zelidauth } })).text(); + return !(await installedNames()).includes(againApp); + }, { timeout: 120000, interval: 3000, label: 'the app is uninstalled' }).catch(() => {}); + installedAfterUninstall = await installedNames(); + await installSeededOnNodes(env, againSpec, [HOLDER]).catch((error) => step(`the install did not settle: ${error.message}`)); + await waitFor(async () => (await installedNames()).includes(againApp), { + timeout: 180000, interval: 2000, label: 'the app is installed again', + }).catch(() => {}); + installedAgain = await installedNames(); + + step('the decision released'); + const releasedFrom = holder.getLastEventId(); + if (holdable) await holder.releaseCheckpoint(BEFORE_DECISION, againApp); + decision = (await heard('boot:appConfirmed', (d) => d.name === againApp, 60000, releasedFrom))?.data ?? null; + removedAfterDecision = Boolean(await heard('app:removed', (d) => d.name === againApp, 30000, releasedFrom)); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await holder?.releaseAllCheckpoints().catch(() => {}); + }); + + it('had its reads of the location records refused by the database', function () { + expect(failedReads, 'the database refused none of the node\'s location reads, so nothing below was tested').to.be.at.least(1); + }); + + it('holds the app\'s decision after reading its expired record', function () { + expect(heldAtBoot, 'boot did not hold the app whose location it could not read').to.equal(true); + expect(parkedAtDecision, 'the decision was not held, so the uninstall and install below did not land before it').to.equal(true); + }); + + it('uninstalls the app and installs it again while its decision is held', function () { + expect(installedAfterUninstall, 'the app was not uninstalled').to.not.include(againApp); + expect(installedAgain, 'the app was not installed again').to.include(againApp); + }); + + it('leaves the app installed again alone when the decision about the uninstalled one is released', async function () { + this.timeout(240000); + expect(decision, 'the held decision never ran, so nothing below was tested').to.not.equal(null); + expect(decision.valid, 'the record boot read was not the expired one').to.equal(false); + expect(removedAfterDecision, 'the decision about the uninstalled app removed the app installed after it').to.equal(false); + expect(await installedNames(), 'the app installed again was removed').to.include(againApp); + await waitFor(() => isUp(again), { timeout: 200000, interval: 3000, label: 'the app installed again runs' }); + }); + }); +}); diff --git a/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js b/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js new file mode 100644 index 0000000000..4d38e57e99 --- /dev/null +++ b/test-infra/runner/tests/3309-a-removal-that-stops-part-way-is-finished.js @@ -0,0 +1,193 @@ +/* + * A hard redeploy whose removal cannot delete the app's row stops with the app part + * removed: its containers and volume are gone, its row is not. The node holds it as + * departing - never announced - and runs the removal again until it finishes, then + * tells the network it no longer holds the app. + * + * The node's deletes of the row are refused REFUSED times: the redeploy's removal and + * the next runs of it, so the app is held across at least one announcement. A second + * app on the node is in every announcement, which shows announcements went out. + * + * While the app is held, nothing installs it: the removal that finishes it would take + * whatever held its name. A second hard redeploy, sent before the removal runs again, + * removes the app and is refused the install. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { getAppContainerId } from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { authenticate } from '../auth.js'; +import { appOwnerKey } from '../framework/keys.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const HOLDER = 0; +// each run of the removal waits 15s after the last, and the node announces every ~30s +const REFUSED = 4; + +function step(what) { + console.log(`# 3309 ${new Date().toISOString()} ${what}`); +} + +const component = (name) => ({ + name, + description: 'part removal test container', + repotag: `${REGISTRY_REPO_HOST}/${name}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', +}); + +describe('a removal that stops part way is finished', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + const appName = `e2epartremoved${stamp}`; + const otherName = `e2epartother${stamp}`; + let holder; + let release; + let anchors; + let failedAt; + + const holds = async (name) => (await holder.getInstalledApps()).data?.some((a) => a.name === name) ?? null; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 3, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await Promise.all([appName, otherName].map((name) => pushImage(name, 'v1'))); + const app = await buildSeedableApp({ env, name: appName, compose: [component(appName)] }); + const other = await buildSeedableApp({ env, name: otherName, compose: [component(otherName)] }); + await installOnNodes(env, app, [HOLDER]); + await installOnNodes(env, other, [HOLDER]); + await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)) + && Boolean(await getAppContainerId(holder.container, otherName, otherName)), { + timeout: 120000, interval: 2000, label: 'both apps run before the redeploy', + }); + + const auth = await authenticate(holder.url, appOwnerKey()); + release = await failCommands(HOLDER + 1, 'localzelapps', 'zelappsinformation', { commands: ['findAndModify'], times: REFUSED }); + step(`hard redeploy with the row delete refused ${REFUSED} times`); + const body = await holder.redeployApp(appName, auth.zelidauth, { force: true }); + if (/Unauthorized/i.test(body)) throw new Error(`hard redeploy refused: ${body.slice(0, 400)}`); + anchors = env.clients.map((client) => client?.getLastEventId()); + failedAt = holder.getLastEventId(); + step('redeploy returned'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('no longer holds the app once its removal finishes', async function () { + this.timeout(240000); + await waitFor(async () => (await holds(appName)) === false, { + timeout: 210000, interval: 2000, label: 'the node no longer holds the app', + }); + }); + + it('had its deletes of the app\'s row refused by the database', async function () { + const failed = await release(); + release = null; + expect(failed, 'the database refused none of the node\'s row deletes, so nothing here was tested').to.be.at.least(1); + }); + + it('never announces the app once its removal has stopped part way', function () { + const announcements = holder.getEventBuffer().filter((e) => e.id > failedAt && e.event === 'app:running'); + expect( + announcements.some((e) => e.data.apps?.some((a) => a.name === otherName)), + 'the node announced nothing while the app was held, so nothing here was tested', + ).to.equal(true); + expect( + announcements.filter((e) => e.data.apps?.some((a) => a.name === appName)).length, + 'the node announced an app it had part removed', + ).to.equal(0); + }); + + it('tells the network it no longer holds the app', async function () { + this.timeout(90000); + const peers = env.clients.filter((client, i) => client && i !== HOLDER); + const heard = await Promise.any(peers.map((peer) => peer.waitForEvent( + 'network:appremoved', + (d) => d.name === appName && d.ip.startsWith(holder.ip), + 60000, + { afterId: anchors[env.clients.indexOf(peer)] }, + ))).catch(() => null); + expect(heard, 'no node was told the app had gone').to.not.equal(null); + }); +}); + +describe('an app whose removal is pending takes no install', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2eheldnoinstall${Date.now()}`; + let holder; + let release; + let afterFailure; + let failedDeletes; + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ hookCtx: this, nodes: 3, tickerAutostart: false }); + await bootAndPeer(env); + holder = env.clients[HOLDER]; + + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ env, name: appName, compose: [component(appName)] }); + await installOnNodes(env, app, [HOLDER]); + await waitFor(async () => Boolean(await getAppContainerId(holder.container, appName, appName)), { + timeout: 120000, interval: 2000, label: 'the app runs before the redeploys', + }); + + const auth = await authenticate(holder.url, appOwnerKey()); + release = await failCommands(HOLDER + 1, 'localzelapps', 'zelappsinformation', { commands: ['findAndModify'], times: 1 }); + step('hard redeploy with the row delete refused once'); + const first = await holder.redeployApp(appName, auth.zelidauth, { force: true }); + if (/Unauthorized/i.test(first)) throw new Error(`hard redeploy refused: ${first.slice(0, 400)}`); + failedDeletes = await release(); + release = null; + afterFailure = holder.getLastEventId(); + step('second hard redeploy while the app is held'); + await holder.redeployApp(appName, auth.zelidauth, { force: true }); + step('second redeploy returned'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('had its delete of the app\'s row refused by the database', function () { + expect(failedDeletes, 'the database refused none of the node\'s row deletes, so nothing here was tested').to.equal(1); + }); + + it('no longer holds the app once its removal finishes', async function () { + this.timeout(120000); + await waitFor(async () => (await holder.getInstalledApps()).data?.some((a) => a.name === appName) === false, { + timeout: 90000, interval: 2000, label: 'the node no longer holds the app', + }); + }); + + it('never installs the app while its removal is pending', function () { + const installed = holder.getEventBuffer().filter((e) => e.id > afterFailure && e.event === 'app:installed' + && e.data.name === appName); + expect(installed.length, 'the node installed an app its pending removal then took').to.equal(0); + }); +}); diff --git a/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js new file mode 100644 index 0000000000..96d52d4fdc --- /dev/null +++ b/test-infra/runner/tests/3310-a-primary-that-went-quiet-and-then-died-is-taken-over.js @@ -0,0 +1,197 @@ +// weight: heavy +/* + * A single-writer app's primary stops announcing while it carries on running: its + * FluxOS is gone, its syncthing and its container are not. + * + * While its location is listed it stays in the standby's folder, across a restart + * of the standby's FluxOS that leaves the standby unable to ask it for its device. + * Once its location expires the standby drops it from the folder, though its + * syncthing is still connected, and what it writes from then on reaches no holder. + * FDM still names it, so the standby stays a standby. + * + * Then the machine dies. Changes on the standby's disk to a file and a directory the + * primary wrote are held by no other device: syncthing reverts such a change by + * deleting it, which a directory that is not empty refuses for ever. The rest of the + * folder is unchanged, so the standby's copy still reads as synced and the revert is + * due. The standby keeps the changes, takes the app over with the primary's data + * intact, and a new standby syncs that copy. + * + * A = the standby, B = the primary, C = the new standby. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { + crashFluxos, execInContainer, releaseFluxos, stopAppContainer, +} from '../framework/container.js'; +import { + getDeviceId, getConnectedDevices, getFolderConfig, getFileInfo, isDaemonUp, readPath, scanFolder, stopDaemon, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { installOnNodes } from '../framework/reconciler-suite.js'; +import { clearMaster } from '../framework/fdm-control.js'; +import { waitFor } from '../framework/wait.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// The harness's syncthing monitor passes every 3s; this is several of them. +const MONITOR_PASSES_MS = 30000; +// A few election passes and a container start. +const TAKEOVER_MS = 10 * 60 * 1000; +// syncthing's FlagLocalReceiveOnly +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const SAVE = 'appdata/save-b'; +const SAVE_CONTENT = 'written by the primary'; +const LISTED = 'appdata/written-while-listed'; +const FENCED = 'appdata/written-after-expiry'; +// Lookups of the primary's device on the standby, each one pass. +const LOOKUP_PASSES = 3; +// Where a node's syncthing monitor took each device it looked up, counted per source. +const DEVICE_SOURCES = ['verified', 'verifiedNoDevice', 'introduced', 'withheld', 'held', 'unsigned', 'configured']; + +function step(what) { + console.log(`# 3310 ${new Date().toISOString()} ${what}`); +} + +describe('a primary that went quiet and then died is taken over', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2equietdied${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let primaryDevice; + let afterDeath; + // How long the primary's write took to reach the standby while it was listed. + let listedSyncMs; + const lookups = async () => { + const counts = await Promise.all(DEVICE_SOURCES.map((source) => client(A) + .getDecisionCount('syncthing:deviceSource', `${client(B).ip}:16127`, source))); + return counts.reduce((sum, n) => sum + n, 0); + }; + async function writeOnPrimary(path, content) { + const wrote = await execInContainer(client(B).container, `printf '${content}' > ${root}/${path}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on the primary: ${wrote.output}`); + await scanFolder(client(B), folder); + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + primaryDevice = await getDeviceId(client(B)); + const wrote = await execInContainer(client(B).container, `printf '${SAVE_CONTENT}' > ${root}/${SAVE}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on the primary: ${wrote.output}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 240000, interval: 3000, label: 'the primary\'s second file reached the standby', + }); + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('keeps a listed primary whose FluxOS does not answer in the folder, across a restart of the standby\'s FluxOS', async function () { + this.timeout(600000); + step('the primary stops announcing; its syncthing and app carry on'); + await crashFluxos(client(B).container, { hold: true }); + expect(await isDaemonUp(client(B)), 'the primary\'s syncthing went down with its FluxOS, so nothing here is tested').to.equal(true); + + step('the standby\'s FluxOS restarts, holding nothing for the primary\'s device'); + await crashFluxos(client(A).container, { hold: true }); + await releaseFluxos(client(A).container); + await waitFor(async () => (await lookups()) >= LOOKUP_PASSES, { + timeout: 240000, interval: 3000, label: `${LOOKUP_PASSES} lookups of the primary's device on the restarted standby`, + }); + const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; + expect(devices, 'the standby dropped a listed primary from the folder once it could not ask it for its device').to.include(primaryDevice); + expect(await client(A).getDecisionCount('syncthing:deviceSource', `${client(B).ip}:16127`, 'configured'), + 'the standby never took the primary\'s device from its own configuration').to.be.at.least(1); + + const startedAt = Date.now(); + await writeOnPrimary(LISTED, 'written while listed'); + await waitFor(async () => (await readPath(client(A), `${root}/${LISTED}`)) === 'written while listed', { + timeout: 120000, interval: 1000, label: 'the listed primary\'s write reached the standby', + }); + listedSyncMs = Date.now() - startedAt; + expect(fleet.twoRan, 'the standby started beside a primary that still runs').to.deep.equal([]); + }); + + it('drops the primary from the folder once its location expires, and what it writes then reaches no holder', async function () { + this.timeout(300000); + const expired = await Promise.all([A, C].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); + expect(expired[0], 'the standby held no location for the primary, so none expired').to.be.at.least(1); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) + .some((d) => d.deviceID === primaryDevice), { + timeout: 120000, interval: 3000, label: 'the standby drops the primary no location lists from the folder', + }); + + step('the primary writes after its location expired'); + await writeOnPrimary(FENCED, 'written after expiry'); + // Connected and fenced: the write would arrive over this connection if the + // folder were still shared with it. + expect(await getConnectedDevices(client(A)), 'the primary\'s syncthing is not connected to the standby').to.include(primaryDevice); + await sleepUnlessInfraDead(Math.max(MONITOR_PASSES_MS, 5 * listedSyncMs)); + expect(await readPath(client(A), `${root}/${FENCED}`), 'the write of a primary no location lists reached the standby').to.equal(null); + expect(fleet.twoRan, 'the standby started beside a primary that still runs').to.deep.equal([]); + }); + + it('takes over once that primary dies, keeping the primary\'s data only it holds', async function () { + this.timeout(TAKEOVER_MS + 300000); + step('the primary\'s machine dies'); + await stopAppContainer(client(B).container, appName, appName); + await stopDaemon(client(B)); + + step('the standby\'s copy of the primary\'s data changes on its disk'); + afterDeath = client(A).getLastEventId(); + const changed = await execInContainer(client(A).container, `chmod 600 ${root}/${SAVE} && chmod 700 ${root}/appdata`); + expect(changed.exitCode, `could not change the standby's copy: ${changed.output}`).to.equal(0); + await scanFolder(client(A), folder); + // eslint-disable-next-line no-bitwise + const heldOnlyHere = (record) => ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; + let record; + await waitFor(async () => { + record = await getFileInfo(client(A), folder, SAVE); + return heldOnlyHere(record) && heldOnlyHere(await getFileInfo(client(A), folder, 'appdata')); + }, { timeout: 60000, interval: 2000, label: 'the file and the directory are changes no other device holds' }); + expect(record.local.version.some((counter) => counter.startsWith(`${primaryDevice.substring(0, 7)}:`)), + 'the changed file carries no history from the primary, so it is not the primary\'s data').to.equal(true); + expect(heldOnlyHere(await getFileInfo(client(A), folder, 'appdata/seed-data')), + 'the unchanged file is held only here too, so the standby\'s copy no longer reads as synced').to.equal(false); + + // FDM still names the dead primary, so the standby stays a standby while its + // passes judge the change; then FDM drops it, as it drops a primary whose app + // stops answering. + await sleepUnlessInfraDead(MONITOR_PASSES_MS); + await clearMaster(appName); + step('waiting for the takeover'); + await waitFor(() => fleet.isUp(A), { + timeout: TAKEOVER_MS, interval: 5000, label: 'the standby takes the app over', + }); + expect(await readPath(client(A), `${root}/${SAVE}`), 'the primary\'s file is gone from the standby').to.equal(SAVE_CONTENT); + expect(await readPath(client(A), `${root}/${FENCED}`), 'the dead primary\'s write from after its location expired reached the standby').to.equal(null); + expect((await readPath(client(A), `${root}/appdata/seed-data`))?.trim(), 'the primary\'s data is gone from the standby').to.equal('seeded'); + const kept = client(A).getEventBuffer().some((e) => e.id > afterDeath && e.event === 'syncthing:localChangesKept' + && e.data?.folder === folder); + expect(kept, 'the standby did not keep the change no other device holds').to.equal(true); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); + + it('gives a new standby the survivor\'s copy', async function () { + this.timeout(600000); + const afterInstall = client(C).getLastEventId(); + await installOnNodes(fleet.env, fleet.app, [C]); + await waitFor(async () => (await readPath(client(C), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 300000, interval: 5000, label: 'the new standby holds the primary\'s data', + }); + await client(C).waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 240000, { afterId: afterInstall }); + expect(await readPath(client(C), `${root}/${FENCED}`), 'the dead primary\'s write from after its location expired reached the new standby').to.equal(null); + }); +}); diff --git a/test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js b/test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js new file mode 100644 index 0000000000..1d8efad6c7 --- /dev/null +++ b/test-infra/runner/tests/3311-a-late-message-whose-registry-row-could-not-be-written-gets-it-next-pass.js @@ -0,0 +1,125 @@ +/* + * A running node fetches in bulk the app messages missing behind its payment records, + * stores them, and then writes the registry row of each app they belong to before it + * marks their hashes found. Here the database refuses the first of those row writes: + * the messages are stored, the rows are not, and the hashes stay unmarked. + * + * A hash whose message the node already stores waits on no peer, so the retry ladder + * that spaces out asking peers does not apply to it: the next pass writes the rows from + * the stored copies. The node must hold every app at its newest message within a few + * passes, and must never have backed off one of those hashes. + */ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { buildSeedableApp, buildSeedableUpdate } from '../framework/seed-helper.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { waitForDaemonReady, waitForOrchestratorState, waitFor } from '../framework/wait.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const NODES = 4; +// the running node: held back until the fleet is up +const SUBJECT = NODES - 1; +const LATE_APPS = 10; +const UPDATES = 54; + +const unresolved = (entry) => ({ ...entry, message: false }); + +function step(what) { + console.log(`# 3311 ${new Date().toISOString()} ${what}`); +} + +describe('a late message whose registry row could not be written gets it on the next pass', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + // the newest message of each app the late log brings, by name + const governing = new Map(); + let lateHashes; + let release; + let failedWrites; + let bulkFetched; + const db = () => dbClient(SUBJECT + 1); + + before(async function () { + this.timeout(900000); + env = await createTestEnv({ + hookCtx: this, + nodes: NODES, + deferredNodes: 1, + // a pass comes every few blocks + configOverrides: { fluxapps: { hashSyncFallbackRecheckBlocks: 2 } }, + }); + await bootAndPeer(env); + const base = env.initialHeight - 2000; + + const subject = await env.startNode(SUBJECT); + await waitForDaemonReady(subject); + await env.startDiscovery([SUBJECT]); + await waitForOrchestratorState(subject, 'READY', 600000); + + // the late log: enough messages to take the bulk path + const late = []; + for (let k = 0; k < LATE_APPS; k += 1) { + const name = `e2eRowLate${k}x${stamp}`; + // eslint-disable-next-line no-await-in-loop + let current = await buildSeedableApp({ env, name, height: base + 100 + (k * 60) }); + late.push(current); + for (let j = 1; j <= UPDATES; j += 1) { + // eslint-disable-next-line no-await-in-loop + current = await buildSeedableUpdate(current, (spec) => { spec.description = `${name} revision ${j}`; }, { height: base + 100 + (k * 60) + j }); + late.push(current); + } + governing.set(name, current.hash); + } + expect(late.length, 'enough messages late to take the bulk path').to.be.above(500); + lateHashes = late.map((m) => m.hashEntry.hash); + + const anchor = subject.getLastEventId(); + step(`seeding ${late.length} late messages, the subject's first registry row write refused`); + release = await failCommands(SUBJECT + 1, 'globalzelapps', 'zelappsinformation', { commands: ['update'], times: 1 }); + await Promise.all(env.clients.map(async (client, i) => { + if (!client || i === SUBJECT) return; + await dbClient(i + 1).seedPermanentMessages(late.map((m) => m.permanentMessage)); + await dbClient(i + 1).seedAppHashes(late.map((m) => m.hashEntry)); + })); + await db().seedAppHashes(late.map((m) => unresolved(m.hashEntry))); + bulkFetched = await subject.waitForEvent('hashSync:bulkFetched', (d) => d.processed > 0, 600000, { afterId: anchor }) + .catch(() => null); + step('fetched in bulk'); + }); + + after(async function () { + this.timeout(60000); + await release?.().catch(() => {}); + await env?.teardown(); + }); + + it('took the late messages in bulk', function () { + expect(bulkFetched, 'the node never fetched the late messages in bulk, so nothing below was tested').to.not.equal(null); + }); + + it('had a registry row write refused by the database', async function () { + failedWrites = await release(); + release = null; + expect(failedWrites, 'the database refused none of the node\'s registry writes, so nothing below was tested').to.equal(1); + }); + + it('holds every late app at its newest message within a few passes', async function () { + this.timeout(180000); + const names = [...governing.keys()]; + await waitFor(async () => { + const held = await Promise.all(names.map(async (name) => (await db().globalAppSpec(name))?.hash)); + return held.every((hash, i) => hash === governing.get(names[i])); + }, { timeout: 150000, interval: 3000, label: 'every late app held at its newest message' }); + }); + + it('never backed off a hash whose message it already stored', async function () { + const records = await db().appHashRecords({ hash: { $in: lateHashes } }); + const backedOff = records.filter((r) => (r.syncAttempts ?? 0) > 0).map((r) => r.hash); + expect(records.length, 'the node holds no record of the late hashes').to.equal(lateHashes.length); + expect(backedOff, 'a hash whose message the node stored was backed off as if no peer had it').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js b/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js new file mode 100644 index 0000000000..ae11c00130 --- /dev/null +++ b/test-infra/runner/tests/3312-a-change-checked-while-the-primary-held-it-is-kept-once-the-primary-is-gone.js @@ -0,0 +1,120 @@ +// weight: heavy +/* + * A standby's copy of a file the primary wrote changes while the primary still holds + * it. Whether the standby's changes hold the only copy of anything is checked off the + * monitor pass, and this check answers that they do not: the primary holds the file. + * + * The check is held before it leaves that answer, and the primary's machine dies and + * is dropped from the folder. From that moment the change is the only copy of the + * primary's data. The answer, released, describes the folder as it was; a revert on + * it deletes that copy. The check the standby asks for next is held before it starts, + * so the old answer is the one its passes find. They revert nothing, and once that + * check runs the standby keeps the change, with the primary's data still on its disk. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { crashFluxos, execInContainer, stopAppContainer } from '../framework/container.js'; +import { + getDeviceId, getFolderConfig, getFileInfo, readPath, scanFolder, stopDaemon, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const BEFORE_ANSWER = 'syncthing:beforeLocalChangeAnswer'; +const BEFORE_CHECK = 'syncthing:beforeLocalChangeCheck'; +const PASSES_ON_THE_OLD_ANSWER = 3; +// syncthing's FlagLocalReceiveOnly +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const SAVE = 'appdata/save-b'; +const SAVE_CONTENT = 'written by the primary'; + +function step(what) { + console.log(`# 3312 ${new Date().toISOString()} ${what}`); +} + +describe('a change checked while the primary held it is kept once the primary is gone', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2echeckedstale${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const passes = () => client(A).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + let primaryDevice; + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + primaryDevice = await getDeviceId(client(B)); + const wrote = await execInContainer(client(B).container, `printf '${SAVE_CONTENT}' > ${root}/${SAVE}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on the primary: ${wrote.output}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 240000, interval: 3000, label: 'the primary\'s file reached the standby', + }); + }); + + after(async function () { + this.timeout(60000); + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('keeps the change, though the check answered while the primary held it', async function () { + this.timeout(1500000); + // eslint-disable-next-line no-bitwise + const heldOnlyHere = (record) => ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; + + step('the standby\'s copy of the primary\'s file changes, while the primary holds it'); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const changedAt = client(A).getLastEventId(); + const changed = await execInContainer(client(A).container, `chmod 600 ${root}/${SAVE}`); + expect(changed.exitCode, `could not change the standby's copy: ${changed.output}`).to.equal(0); + await scanFolder(client(A), folder); + expect(heldOnlyHere(await getFileInfo(client(A), folder, SAVE)), 'fixture: the primary does not hold the file').to.equal(false); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_ANSWER && d.key === folder, 300000, { afterId: changedAt }); + + step('the primary\'s machine dies and the standby drops it from the folder'); + await crashFluxos(client(B).container, { hold: true }); + await Promise.all([A, 2].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); + await stopAppContainer(client(B).container, appName, appName); + await stopDaemon(client(B)); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []) + .some((d) => d.deviceID === primaryDevice), { + timeout: 600000, interval: 5000, label: 'the standby drops the dead primary from the folder', + }); + await waitFor(async () => heldOnlyHere(await getFileInfo(client(A), folder, SAVE)), { + timeout: 60000, interval: 2000, label: 'fixture: the change is now the only copy', + }); + + step('the check, answered before the primary left, is released; the next is held before it starts'); + await client(A).holdCheckpoint(BEFORE_CHECK, folder); + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + await client(A).waitForEvent('syncthing:localChangesChecked', (d) => d.folder === folder && d.onlyCopy === false, 300000, { afterId: releasedAt }); + const passesOnOldAnswer = await passes(); + await waitFor(async () => await passes() >= passesOnOldAnswer + PASSES_ON_THE_OLD_ANSWER, { + timeout: 120000, interval: 2000, label: `${PASSES_ON_THE_OLD_ANSWER} passes with the answer from before the primary left`, + }); + const revertedOnOldAnswer = client(A).getEventBuffer().some((e) => e.id > releasedAt && e.event === 'syncthing:localChangesReverted' && e.data?.folder === folder); + expect(revertedOnOldAnswer, 'the standby reverted the only copy on the answer from before the primary left').to.equal(false); + + step('the check asked for since runs'); + await client(A).releaseCheckpoint(BEFORE_CHECK, folder); + await client(A).waitForEvent('syncthing:localChangesKept', (d) => d.folder === folder, 300000, { afterId: releasedAt }); + + const reverted = client(A).getEventBuffer().some((e) => e.id > changedAt && e.event === 'syncthing:localChangesReverted' && e.data?.folder === folder); + expect(reverted, 'the standby reverted the only copy on an answer from before the primary left').to.equal(false); + expect(await readPath(client(A), `${root}/${SAVE}`), 'the primary\'s file is gone from the standby').to.equal(SAVE_CONTENT); + }); +}); diff --git a/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js new file mode 100644 index 0000000000..f0f66d653e --- /dev/null +++ b/test-infra/runner/tests/3313-a-standbys-local-changes-are-checked-off-the-pass.js @@ -0,0 +1,140 @@ +// weight: heavy +/* + * Whether a standby's local changes hold the only copy of the primary's data is + * checked off the monitor pass, and acted on only while the folder is the one it was + * checked against. The primary is alive throughout, so every change here is held + * elsewhere and is reverted in the end; what is tested is when. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { crashFluxos, releaseFluxos, execInContainer } from '../framework/container.js'; +import { readPath, scanFolder } from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const BEFORE_ANSWER = 'syncthing:beforeLocalChangeAnswer'; +const BEFORE_CHECK = 'syncthing:beforeLocalChangeCheck'; +const FILES = 2000; +const PASSES_WHILE_HELD = 3; + +function step(what) { + console.log(`# 3313 ${new Date().toISOString()} ${what}`); +} + +describe('a standby\'s local changes are checked off the monitor pass', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2echeckoffpass${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const passes = () => client(A).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + const heldAt = (from) => client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_ANSWER && d.key === folder, 300000, { afterId: from }); + const event = (name, from) => client(A).waitForEvent(name, (d) => d.folder === folder, 300000, { afterId: from }); + const eventsSince = (name, from) => client(A).getEventBuffer().filter((e) => e.id > from && e.event === name && e.data?.folder === folder); + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + await sh(B, `mkdir -p ${root}/appdata/many && for i in $(seq 1 ${FILES}); do printf 'primary %s' $i > ${root}/appdata/many/f$i; done`); + await scanFolder(client(B), folder); + await waitFor(async () => (await sh(A, `ls ${root}/appdata/many | wc -l`)).trim() === String(FILES), { + timeout: 600000, interval: 5000, label: 'the primary\'s files reached the standby', + }); + }); + + // A test that fails holding a check would hold the next test's checks too. + afterEach(async function () { + await client(A)?.releaseAllCheckpoints().catch(() => {}); + }); + + after(async function () { + this.timeout(60000); + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('runs the standby\'s passes while it checks a rewrite of every file the primary wrote, then reverts it', async function () { + this.timeout(900000); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const from = client(A).getLastEventId(); + step(`the standby rewrites its copy of ${FILES} of the primary's files`); + await sh(A, `for i in $(seq 1 ${FILES}); do printf ' standby' >> ${root}/appdata/many/f$i; done`); + await scanFolder(client(A), folder); + await heldAt(from); + + const passesAtHold = await passes(); + await waitFor(async () => await passes() >= passesAtHold + PASSES_WHILE_HELD, { + timeout: 120000, interval: 2000, label: `${PASSES_WHILE_HELD} passes over the folder while its check is held`, + }); + expect(eventsSince('syncthing:localChangesReverted', from), 'reverted before the check answered').to.deep.equal([]); + + await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + await event('syncthing:localChangesReverted', from); + await waitFor(async () => (await readPath(client(A), `${root}/appdata/many/f1`)) === 'primary 1', { + timeout: 120000, interval: 2000, label: 'the standby holds the primary\'s copy again', + }); + }); + + it('does not revert on an answer from before a later write, and reverts once that write is checked', async function () { + this.timeout(900000); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const from = client(A).getLastEventId(); + await sh(A, `printf ' first' >> ${root}/appdata/many/f1`); + await scanFolder(client(A), folder); + await heldAt(from); + + step('a second write lands while the first is being checked'); + await sh(A, `printf ' second' >> ${root}/appdata/many/f2`); + await scanFolder(client(A), folder); + const passesAfterWrite = await passes(); + await waitFor(async () => await passes() >= passesAfterWrite + 2, { + timeout: 120000, interval: 2000, label: 'two passes have seen the second write', + }); + + // The check asked for since the second write is held before it starts, so the + // passes find the answer from before it. + await client(A).holdCheckpoint(BEFORE_CHECK, folder); + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_ANSWER, folder); + await event('syncthing:localChangesChecked', releasedAt); + const passesOnOldAnswer = await passes(); + await waitFor(async () => await passes() >= passesOnOldAnswer + 3, { + timeout: 120000, interval: 2000, label: 'three passes with the answer from before the second write', + }); + expect(eventsSince('syncthing:localChangesReverted', from), 'reverted on the answer from before the second write').to.deep.equal([]); + + await client(A).releaseCheckpoint(BEFORE_CHECK, folder); + await event('syncthing:localChangesReverted', from); + }); + + it('checks again after FluxOS restarts mid-check, and reverts', async function () { + this.timeout(900000); + await client(A).holdCheckpoint(BEFORE_ANSWER, folder); + const from = client(A).getLastEventId(); + await sh(A, `printf ' again' >> ${root}/appdata/many/f3`); + await scanFolder(client(A), folder); + await heldAt(from); + + step('the standby\'s FluxOS restarts with the check held'); + await crashFluxos(client(A).container, { hold: true }); + await releaseFluxos(client(A).container); + await waitFor(async () => (await readPath(client(A), `${root}/appdata/many/f3`)) === 'primary 3', { + timeout: 600000, interval: 5000, label: 'the restarted standby reverts to the primary\'s copy', + }); + }); +}); diff --git a/test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js b/test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js new file mode 100644 index 0000000000..5299172ca0 --- /dev/null +++ b/test-infra/runner/tests/3314-a-former-primary-left-the-only-holder-keeps-what-it-wrote-as-primary.js @@ -0,0 +1,147 @@ +// weight: heavy +/* + * A node that was a single-writer app's primary, and later its standby, is left the + * only holder. What it wrote as the primary carries no other device's history, which + * is also what a standby's own junk carries; but no other holder runs the app, so + * this node holds its data, and a revert has nothing to restore it from. Its own + * earlier file and directory change on its disk: it keeps both and takes the app + * over with them. + * + * Reverted, the file is deleted - no other device holds it - and the directory, not + * empty, can never be: the folder is reverted on every pass and never taken over. + * + * A = the standby, then the primary; B = the primary, then the standby; C = holds + * nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { + crashFluxos, releaseFluxos, execInContainer, stopAppContainer, +} from '../framework/container.js'; +import { + getDeviceId, getFolderConfig, getFileInfo, readPath, scanFolder, startDaemon, stopDaemon, waitForDaemonEvent, folderSaved, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { clearMaster, electMaster } from '../framework/fdm-control.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// A machine that dies closes none of its connections, so a holder's syncthing keeps +// them until its own receive timeout (5 min). Then a few election passes and a start. +const TAKEOVER_MS = 10 * 60 * 1000; +// syncthing's FlagLocalReceiveOnly +const RECEIVE_ONLY_CHANGE_FLAG = 8; +const FILE = 'appdata/written-as-primary.txt'; +const DIR = 'appdata/primary-dir'; +const CONTENT = 'written by B as the primary'; + +function step(what) { + console.log(`# 3314 ${new Date().toISOString()} ${what}`); +} + +describe('a former primary left the only holder keeps what it wrote as the primary', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2eformerprimary${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + // eslint-disable-next-line no-bitwise + const heldOnlyHere = (record) => ((record?.global?.localFlags ?? 0) & RECEIVE_ONLY_CHANGE_FLAG) !== 0; + const historyOf = async (i, item) => (await getFileInfo(client(i), folder, item))?.local?.version ?? []; + + // The machine of node `dead` dies, and `holder` drops it from the folder. + async function machineDies(dead, holder) { + const deadDevice = await getDeviceId(client(dead)); + await crashFluxos(client(dead).container, { hold: true }); + await Promise.all([A, B, C].filter((i) => i !== dead) + .map((i) => dbClient(i + 1).expireAppLocation(appName, client(dead).ip))); + await stopAppContainer(client(dead).container, appName, appName); + await stopDaemon(client(dead)); + await waitFor(async () => !((await getFolderConfig(client(holder), folder))?.devices ?? []) + .some((d) => d.deviceID === deadDevice), { + timeout: TAKEOVER_MS, interval: 5000, label: `node ${holder} drops node ${dead} from the folder`, + }); + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + await sh(B, `mkdir -p ${root}/${DIR} && printf '${CONTENT}' > ${root}/${FILE} && printf 'inside' > ${root}/${DIR}/inside.txt`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), `${root}/${FILE}`)) === CONTENT, { + timeout: 240000, interval: 3000, label: 'what the primary wrote reached the standby', + }); + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('hands the app to the standby when the primary dies', async function () { + this.timeout(TAKEOVER_MS + 300000); + step('B, the primary, dies'); + await machineDies(B, A); + await clearMaster(appName); + await waitFor(() => fleet.isUp(A), { timeout: TAKEOVER_MS, interval: 5000, label: 'A takes the app over' }); + await electMaster(appName, client(A).ip); + }); + + it('brings B back as a standby holding what it wrote as the primary', async function () { + this.timeout(600000); + step('B returns'); + await startDaemon(client(B), { paused: true }); + await releaseFluxos(client(B).container); + await waitForDaemonEvent(client(B), folderSaved(folder, { type: 'receiveonly', paused: false }), { + timeout: 300000, label: 'B\'s folder receiving', + }); + expect(await readPath(client(B), `${root}/${FILE}`), 'B holds what it wrote').to.equal(CONTENT); + const ownShortId = (await getDeviceId(client(B))).substring(0, 7); + expect((await historyOf(B, FILE)).every((counter) => counter.startsWith(`${ownShortId}:`)), + 'fixture: the file\'s history names a device other than B, so this is not B\'s own data').to.equal(true); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); + + it('keeps B\'s changes to what it wrote as the primary once it is the only holder, and B takes the app over with them', async function () { + this.timeout(TAKEOVER_MS * 2); + step('A, now the primary, dies'); + await machineDies(A, B); + + step('B\'s copy of its own file and directory changes on its disk'); + const changedAt = client(B).getLastEventId(); + await sh(B, `chmod 600 ${root}/${FILE} && chmod 700 ${root}/${DIR}`); + await scanFolder(client(B), folder); + await waitFor(async () => heldOnlyHere(await getFileInfo(client(B), folder, FILE)), { + timeout: 120000, interval: 2000, label: 'fixture: B\'s change is the only copy', + }); + + // Whichever B decides first: kept or reverted. + const decisions = () => client(B).getEventBuffer().filter((e) => e.id > changedAt && e.data?.folder === folder + && (e.event === 'syncthing:localChangesKept' || e.event === 'syncthing:localChangesReverted')); + await waitFor(async () => decisions().length > 0, { + timeout: 300000, interval: 2000, label: 'B decides what to do with its changes', + }); + expect(decisions()[0].event, 'B reverted the only copy of what it wrote as the primary').to.equal('syncthing:localChangesKept'); + expect(decisions()[0].data.reason, 'why B kept its changes').to.equal('noHolderRunsTheApp'); + expect(await readPath(client(B), `${root}/${FILE}`), 'B\'s file is gone').to.equal(CONTENT); + + step('B takes the app over'); + await electMaster(appName, client(B).ip); + await waitFor(() => fleet.isUp(B), { timeout: TAKEOVER_MS, interval: 5000, label: 'B takes the app over' }); + expect(await readPath(client(B), `${root}/${FILE}`), 'B took the app over without its file').to.equal(CONTENT); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js b/test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js new file mode 100644 index 0000000000..1e33dc18fe --- /dev/null +++ b/test-infra/runner/tests/3315-a-standbys-own-changes-kept-while-no-holder-ran-the-app-are-reverted-once-one-does.js @@ -0,0 +1,99 @@ +// weight: heavy +/* + * A standby's own changes are reverted only while another holder runs the app. The + * primary's machine dies, and a change the standby made itself is kept: no holder + * runs the app, so nothing says it is junk. The primary comes back and runs the app + * again, and the standby, asking again on every pass, reverts the change: the + * primary is authoritative and does not have it. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { + crashFluxos, releaseFluxos, execInContainer, stopAppContainer, +} from '../framework/container.js'; +import { + getConnectedDevices, getDeviceId, readPath, scanFolder, startDaemon, stopDaemon, +} from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// A machine that dies closes none of its connections, so the standby's syncthing +// holds them until its own receive timeout (5 min). +const DISCONNECT_MS = 10 * 60 * 1000; +const PASSES_AFTER_RETURN = 3; +const STRAY = 'appdata/made-on-the-standby.txt'; + +function step(what) { + console.log(`# 3315 ${new Date().toISOString()} ${what}`); +} + +describe('a standby\'s own changes kept while no holder ran the app are reverted once one does', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2ekeptthenreverted${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const passes = () => client(A).getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + const eventsSince = (name, from) => client(A).getEventBuffer().filter((e) => e.id > from && e.event === name && e.data?.folder === folder); + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('keeps a change the standby made while no holder runs the app', async function () { + this.timeout(DISCONNECT_MS + 300000); + step('the primary\'s machine dies; FDM still names it'); + const primaryDevice = await getDeviceId(client(B)); + await crashFluxos(client(B).container, { hold: true }); + await stopAppContainer(client(B).container, appName, appName); + await stopDaemon(client(B)); + await waitFor(async () => !(await getConnectedDevices(client(A))).includes(primaryDevice), { + timeout: DISCONNECT_MS, interval: 5000, label: 'the standby\'s syncthing loses the primary', + }); + + step('the standby makes a change of its own'); + const from = client(A).getLastEventId(); + const wrote = await execInContainer(client(A).container, `printf 'standby' > ${root}/${STRAY}`); + expect(wrote.exitCode, `fixture: ${wrote.output}`).to.equal(0); + await scanFolder(client(A), folder); + const kept = await client(A).waitForEvent('syncthing:localChangesKept', (d) => d.folder === folder, 300000, { afterId: from }); + expect(kept.data.reason, 'why the standby kept its change').to.equal('noHolderRunsTheApp'); + expect(eventsSince('syncthing:localChangesReverted', from), 'the standby reverted while no holder ran the app').to.deep.equal([]); + expect(await readPath(client(A), `${root}/${STRAY}`), 'the standby\'s change').to.equal('standby'); + }); + + it('reverts that change once the primary runs the app again', async function () { + this.timeout(900000); + step('the primary comes back and runs the app'); + const from = client(A).getLastEventId(); + await startDaemon(client(B), { paused: true }); + await releaseFluxos(client(B).container); + await waitFor(() => fleet.isUp(B), { timeout: 600000, interval: 5000, label: 'the primary runs the app again' }); + + const passesWithPrimaryUp = await passes(); + await waitFor(async () => await passes() >= passesWithPrimaryUp + PASSES_AFTER_RETURN, { + timeout: 180000, interval: 2000, label: `${PASSES_AFTER_RETURN} of the standby's passes with the primary running`, + }); + expect(eventsSince('syncthing:localChangesReverted', from).length, 'the standby kept its change beside a primary running the app').to.be.at.least(1); + await waitFor(async () => (await readPath(client(A), `${root}/${STRAY}`)) === null, { + timeout: 120000, interval: 2000, label: 'the standby\'s change is gone', + }); + expect(fleet.twoRan, 'two nodes ran the app at once').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js b/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js new file mode 100644 index 0000000000..476da79956 --- /dev/null +++ b/test-infra/runner/tests/3316-a-primary-no-location-lists-is-fenced-while-it-runs.js @@ -0,0 +1,129 @@ +// weight: heavy +/* + * A single-writer app's primary stops announcing while it carries on running: its + * FluxOS is gone, its syncthing and its container are not. Its location expires. + * + * The standby drops it from the folder though its syncthing is still connected, so + * what it writes from then on reaches no holder. The standby starts nothing beside + * it while that connection is up, though FDM no longer names it: it may still be + * running the app. A holder placed after the location expired syncs the standby's + * copy and nothing the old primary wrote since. + * + * A = the standby, B = the primary, C = the holder placed later. + */ +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { crashFluxos, execInContainer } from '../framework/container.js'; +import { + getDeviceId, getConnectedDevices, getFolderConfig, isDaemonUp, readPath, scanFolder, +} from '../framework/syncthing-real.js'; +import { dbClient } from '../framework/db-client.js'; +import { installOnNodes } from '../framework/reconciler-suite.js'; +import { clearMaster } from '../framework/fdm-control.js'; +import { waitFor } from '../framework/wait.js'; +import { sleepUnlessInfraDead } from '../framework/infra-death.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +// The harness's syncthing monitor passes every 3s; this is several of them. +const MONITOR_PASSES_MS = 30000; +// Several election passes, each of which would start the app on the standby +// if it read the old primary as gone. +const ELECTION_PASSES_MS = 3 * 60 * 1000; +const SAVE = 'appdata/save-b'; +const SAVE_CONTENT = 'written by the primary'; +const FENCED = 'appdata/written-after-expiry'; +const FENCED_LATER = 'appdata/written-before-a-holder-is-placed'; + +function step(what) { + console.log(`# 3316 ${new Date().toISOString()} ${what}`); +} + +describe('a primary no location lists is fenced while it runs', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2efenced${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let primaryDevice; + // How long the primary's write took to reach the standby while it was listed. + let listedSyncMs; + + async function write(i, path, content) { + const wrote = await execInContainer(client(i).container, `printf '${content}' > ${root}/${path}`); + if (wrote.exitCode !== 0) throw new Error(`could not write on node ${i}: ${wrote.output}`); + await scanFolder(client(i), folder); + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + primaryDevice = await getDeviceId(client(B)); + const startedAt = Date.now(); + await write(B, SAVE, SAVE_CONTENT); + await waitFor(async () => (await readPath(client(A), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 240000, interval: 1000, label: 'the primary\'s second file reached the standby', + }); + listedSyncMs = Date.now() - startedAt; + }); + + after(async function () { + this.timeout(60000); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('drops the primary from the standby\'s folder once its location expires, though it still runs', async function () { + this.timeout(300000); + step('the primary stops announcing; its syncthing and app carry on'); + await crashFluxos(client(B).container, { hold: true }); + expect(await isDaemonUp(client(B)), 'the primary\'s syncthing went down with its FluxOS, so nothing here is tested').to.equal(true); + const expired = await Promise.all([A, C].map((i) => dbClient(i + 1).expireAppLocation(appName, client(B).ip))); + expect(expired[0], 'the standby held no location for the primary, so none expired').to.be.at.least(1); + await sleepUnlessInfraDead(MONITOR_PASSES_MS); + + step('the primary writes after its location expired'); + await write(B, FENCED, 'written after expiry'); + await sleepUnlessInfraDead(Math.max(MONITOR_PASSES_MS, 5 * listedSyncMs)); + expect(await readPath(client(A), `${root}/${FENCED}`), 'the write of a primary no location lists reached the standby').to.equal(null); + const devices = (await getFolderConfig(client(A), folder))?.devices?.map((d) => d.deviceID) ?? []; + expect(devices, 'the standby kept a primary no location lists in the folder').to.not.include(primaryDevice); + // Connected and fenced: the write would have arrived over this connection. + expect(await getConnectedDevices(client(A)), 'the primary\'s syncthing is not connected to the standby').to.include(primaryDevice); + expect(await fleet.isUp(B), 'the primary\'s app stopped, so nothing here is tested').to.equal(true); + }); + + it('starts nothing on the standby while the old primary\'s syncthing is connected, though FDM no longer names it', async function () { + this.timeout(ELECTION_PASSES_MS + 120000); + const before = await fleet.verdicts(A, 'connectionAlive'); + await clearMaster(appName); + step('watching the standby\'s election'); + await sleepUnlessInfraDead(ELECTION_PASSES_MS); + expect(await fleet.verdicts(A, 'connectionAlive'), + 'the standby never judged the old primary\'s silence, so nothing here is tested').to.be.above(before); + expect(await fleet.isUp(A), 'the standby started the app beside the old primary still running it').to.equal(false); + expect(await fleet.isUp(B), 'the old primary\'s app stopped, so nothing here is tested').to.equal(true); + }); + + it('gives a holder placed after the expiry the standby\'s copy and nothing the old primary wrote since', async function () { + this.timeout(900000); + await write(B, FENCED_LATER, 'written before a holder is placed'); + + step('a holder is placed after the old primary\'s location expired'); + await installOnNodes(fleet.env, fleet.app, [C]); + await waitFor(async () => (await readPath(client(C), `${root}/${SAVE}`)) === SAVE_CONTENT, { + timeout: 300000, interval: 5000, label: 'the standby\'s copy reached the holder placed later', + }); + await sleepUnlessInfraDead(Math.max(MONITOR_PASSES_MS, 5 * listedSyncMs)); + expect(await readPath(client(C), `${root}/${FENCED}`), 'the old primary\'s write after its location expired reached the holder placed later').to.equal(null); + expect(await readPath(client(C), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the holder placed later').to.equal(null); + expect(await readPath(client(A), `${root}/${FENCED_LATER}`), 'the old primary\'s later write reached the standby').to.equal(null); + expect(await fleet.isUp(A), 'the standby started the app beside the old primary still running it').to.equal(false); + }); +}); diff --git a/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js new file mode 100644 index 0000000000..af46c31370 --- /dev/null +++ b/test-infra/runner/tests/3317-a-standby-reverts-its-changes-while-the-primary-keeps-writing.js @@ -0,0 +1,186 @@ +// weight: heavy +/* + * A standby's local changes are reverted while its primary keeps writing. Every pull + * moves the folder's sequences, and none moves syncthing's counts of the standby's own + * changes, so a check of them stands while the primary writes. + * + * A revert is of the changes checked and no others: one held before it reads the + * folder's changes again, while a further change lands, does not go ahead until the + * folder is checked again. + * + * And it is of the folder checked: one held while the primary's location expires and + * the primary leaves the standby's folder, which makes the standby's change to the + * primary's file the only copy, does not go ahead, and that change is kept. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, +} from '../framework/silent-primary.js'; +import { execInContainer, crashFluxos } from '../framework/container.js'; +import { dbClient } from '../framework/db-client.js'; +import { + readPath, scanFolder, getFolderConfig, getDeviceId, +} from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const BEFORE_REVERT = 'syncthing:beforeLocalChangeRevert'; +// The primary writes this often for the whole suite: faster than the standby's passes. +const WRITE_EVERY_MS = 2000; +// Long enough for many passes, each finding the folder pulled since the one before. +const REVERT_MS = 5 * 60 * 1000; + +function step(what) { + console.log(`# 3317 ${new Date().toISOString()} ${what}`); +} + +describe('a standby reverts its local changes while the primary keeps writing', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2ewriting${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + let writing = false; + let writer = null; + let lines = 0; + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + const linesOnStandby = async () => Number((await sh(A, `wc -l < ${root}/appdata/live.log 2>/dev/null || echo 0`)).trim()); + const eventsSince = (name, from) => client(A).getEventBuffer().filter((e) => e.id > from && e.event === name && e.data?.folder === folder); + const event = (name, from, timeout) => client(A).waitForEvent(name, (d) => d.folder === folder, timeout, { afterId: from }); + + // The primary appends a line and has syncthing scan it, every WRITE_EVERY_MS. + async function keepWriting() { + while (writing) { + lines += 1; + // eslint-disable-next-line no-await-in-loop + await sh(B, `echo line-${lines} >> ${root}/appdata/live.log`).catch(() => {}); + // eslint-disable-next-line no-await-in-loop + await scanFolder(client(B), folder).catch(() => {}); + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, WRITE_EVERY_MS); }); + } + } + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + writing = true; + writer = keepWriting(); + await waitFor(async () => (await linesOnStandby()) >= 3, { + timeout: 240000, interval: 2000, label: 'the primary\'s writes reach the standby', + }); + }); + + // A test that fails holding a revert would hold the next test's reverts too. + afterEach(async function () { + await client(A)?.releaseAllCheckpoints().catch(() => {}); + }); + + after(async function () { + this.timeout(60000); + writing = false; + await writer; + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('reverts its changes while the primary keeps writing', async function () { + this.timeout(REVERT_MS + 120000); + const from = client(A).getLastEventId(); + step('the standby changes the mode of one of the primary\'s files and makes a stray'); + await sh(A, `chmod 600 ${root}/appdata/seed-data && echo stray > ${root}/appdata/stray-1`); + await scanFolder(client(A), folder); + const linesAtChange = await linesOnStandby(); + + await event('syncthing:localChangesReverted', from, REVERT_MS).catch(() => { + throw new Error('the standby did not revert its changes while the primary kept writing'); + }); + // A revert resets syncthing's index; the pull that follows puts the primary's + // version back. + await waitFor(async () => (await readPath(client(A), `${root}/appdata/stray-1`)) === null, { + timeout: 120000, interval: 2000, label: 'the stray is gone from the standby', + }); + await waitFor(async () => (await sh(A, `stat -c %a ${root}/appdata/seed-data`)).trim() !== '600', { + timeout: 120000, interval: 2000, label: 'the primary\'s file has the primary\'s mode again', + }); + await waitFor(async () => (await linesOnStandby()) > linesAtChange, { + timeout: 60000, interval: 2000, label: 'the primary\'s writes kept reaching the standby', + }); + }); + + it('does not revert a change made after the check until it has been checked', async function () { + this.timeout(REVERT_MS + 300000); + await client(A).holdCheckpoint(BEFORE_REVERT, folder); + const from = client(A).getLastEventId(); + await sh(A, `echo stray > ${root}/appdata/stray-2`); + await scanFolder(client(A), folder); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_REVERT && d.key === folder, REVERT_MS, { afterId: from }); + + step('a further change lands while the revert of the first is held'); + await sh(A, `echo stray > ${root}/appdata/stray-3`); + await scanFolder(client(A), folder); + await waitFor(async () => Number((await sh(A, `ls ${root}/appdata | grep -c '^stray-'`)).trim()) === 2, { + timeout: 30000, interval: 1000, label: 'both strays are on the standby', + }); + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_REVERT, folder); + + const reverted = await event('syncthing:localChangesReverted', releasedAt, REVERT_MS); + const checked = eventsSince('syncthing:localChangesChecked', releasedAt); + expect(checked.filter((e) => e.id < reverted.id), 'reverted changes it had not checked').to.not.deep.equal([]); + await waitFor(async () => (await readPath(client(A), `${root}/appdata/stray-2`)) === null + && (await readPath(client(A), `${root}/appdata/stray-3`)) === null, { + timeout: 120000, interval: 2000, label: 'both strays are gone from the standby', + }); + }); + + // Last: the primary is fenced from here on. + it('keeps a change to the primary\'s file whose revert was held while the primary left the folder', async function () { + this.timeout(REVERT_MS + 600000); + writing = false; + await writer; + const kept = `${root}/appdata/kept`; + await sh(B, `printf primary > ${kept}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), kept)) === 'primary', { + timeout: 120000, interval: 2000, label: 'the primary\'s file reached the standby', + }); + + await client(A).holdCheckpoint(BEFORE_REVERT, folder); + const from = client(A).getLastEventId(); + step('the standby changes the primary\'s file, and its revert is held'); + await sh(A, `printf standby > ${kept}`); + await scanFolder(client(A), folder); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === BEFORE_REVERT && d.key === folder, REVERT_MS, { afterId: from }); + + step('the primary stops announcing, its location expires, and it leaves the standby\'s folder'); + const primaryDevice = await getDeviceId(client(B)); + await crashFluxos(client(B).container, { hold: true }); + const expired = await dbClient(A + 1).expireAppLocation(appName, client(B).ip); + expect(expired, 'fixture: the standby held no location for the primary, so none expired').to.be.at.least(1); + await waitFor(async () => !((await getFolderConfig(client(A), folder))?.devices ?? []).some((d) => d.deviceID === primaryDevice), { + timeout: 300000, interval: 2000, label: 'the primary left the standby\'s folder', + }); + + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(BEFORE_REVERT, folder); + const onlyCopy = await client(A).waitForEvent('syncthing:localChangesChecked', (d) => d.folder === folder && d.onlyCopy, REVERT_MS, { afterId: releasedAt }) + .catch(() => null); + + expect(eventsSince('syncthing:localChangesReverted', releasedAt), 'reverted the standby\'s change once it was the only copy').to.deep.equal([]); + expect(await readPath(client(A), kept), 'the standby\'s change to the primary\'s file').to.equal('standby'); + expect(onlyCopy, 'the standby never checked the folder again and found its change the only copy').to.not.equal(null); + }); +}); diff --git a/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js b/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js new file mode 100644 index 0000000000..d458b59b61 --- /dev/null +++ b/test-infra/runner/tests/3318-a-stalled-host-kills-a-container-made-before-the-app-slices.js @@ -0,0 +1,135 @@ +// base-image: FLUX_E2E_BASE_TAG +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer } from '../framework/container.js'; +import { waitForUp, waitForAppInstalled } from '../framework/wait.js'; +import { bootAndPeer, seedSpawnerApp } from '../framework/reconciler-suite.js'; +import { buildSeedableTestApp } from '../framework/seed-helper.js'; +import { pushTestApp } from '../framework/registry-helper.js'; +import { + DOCKER_ROOT, stallDisk, resumeDisk, readsHeld, thawEveryNode, containerCgroup, processesThatCanRun, +} from '../framework/host-stall.js'; +import { failCommands } from '../framework/mongo-control.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node upgraded from a release that gave app containers no slice of their own: +// the app's container, made by the release, lives in its runtime's scope under +// system.slice, and only docker's records on disk say which app it is. The node's +// docker disk then stops answering reads, so those records cannot be read, and its +// database refuses the read of its installed apps, as one on the same disk does, so +// the node finds no app by name. It goes out of service for the stall, and kills +// every container outside the slices all the same: the app's container among them, +// and nothing of those containers is reported as able to run. +// +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming the build it needs. + +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; +const READ_INTERVAL_MS = 5 * 1000; +const STUCK_INTERVALS = 6; +const STOP_WINDOW_MS = 10 * 1000; +const REMOVAL_DEADLINE_MS = 45 * 1000; +// what the node takes beyond the window to check and kill every app +const KILL_SLACK_MS = 10 * 1000; +const STALL_MS = READ_INTERVAL_MS * STUCK_INTERVALS; +const UPGRADED = 0; + +const outOfServiceForTheStall = (data) => data.dosState >= 100 && String(data.dosMessage).includes('Host stall'); +const containerOf = (appName) => `flux${appName}_${appName}`; + +// What of a container's cgroup can still run; a cgroup that has gone holds nothing. +async function canRun(container, cgroupDir) { + const exists = await execInContainer(container, `test -d '${cgroupDir}'`); + return exists.exitCode === 0 ? processesThatCanRun(container, cgroupDir) : []; +} + +describe('a stalled host kills a container made before the app slices', function () { + let env; + dumpLogsOnFailure(() => env); + const appName = `e2eprestall${Date.now()}`; + let client; + let container; + let scope; + let runningBefore; + let readsWereHeld; + let trippedAt; + let stopped; + let runnableAfterWindow; + let releaseDatabase = null; + + before(async function () { + this.timeout(900000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the released FluxOS the node upgrades from: build it with FLUX_E2E_TAG= ./test-infra/build-release-image.sh '); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + nodeImageTags: { [UPGRADED]: BASE_TAG }, + tickerAutostart: false, + freezableDockerRoot: [UPGRADED], + configOverrides: { + hostStall: { readIntervalMs: READ_INTERVAL_MS, stuckIntervals: STUCK_INTERVALS, stopWindowMs: STOP_WINDOW_MS }, + fluxapps: { appRemovalDeadlineMs: REMOVAL_DEADLINE_MS }, + }, + }); + if (BASE_TAG === env.nodeImageTag(1)) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG: the suite upgrades from the one to the other'); + await bootAndPeer(env); + client = env.clients[UPGRADED]; + ({ container } = client); + + // an app that writes a log line every 200 ms, so docker writes while its disk stalls + await pushTestApp(appName); + const app = await buildSeedableTestApp({ name: appName, logEveryMs: 200, instances: 1 }); + await seedSpawnerApp(env, app, [UPGRADED]); + await waitForAppInstalled(client, appName, 180000); + await waitForUp(client, appName, 'the app is up on the release'); + scope = await containerCgroup(container, containerOf(appName)); + + await env.upgradeNode(UPGRADED, { tag: env.nodeImageTag(1) }); + await waitForUp(client, appName, 'the app is up after the upgrade'); + runningBefore = await canRun(container, scope); + + const anchor = client.getLastEventId(); + releaseDatabase = await failCommands(UPGRADED + 1, 'localzelapps', 'zelappsinformation', { commands: ['find'] }); + await stallDisk(container, DOCKER_ROOT); + readsWereHeld = await readsHeld(container, `${DOCKER_ROOT}/containers`, { file: 'config.v2.json' }); + const trip = await client.waitForEvent('dos:changed', outOfServiceForTheStall, STALL_MS + READ_INTERVAL_MS * 4, { afterId: anchor }) + .catch(() => null); + trippedAt = trip ? Date.now() : null; + // the records' read waits one window, and the stop another + stopped = (await client.waitForEvent('hostStall:appsStopped', () => true, STOP_WINDOW_MS * 2 + KILL_SLACK_MS, { afterId: anchor }) + .catch(() => null))?.data ?? null; + runnableAfterWindow = await canRun(container, scope); + }); + + after(async function () { + this.timeout(60000); + await releaseDatabase?.().catch(() => {}); + if (container) await resumeDisk(container, DOCKER_ROOT).catch(() => {}); + if (client) await thawEveryNode([client]); + await env?.teardown(); + }); + + it('runs the app in a container the release made outside any app slice, still running after the upgrade', function () { + expect(scope, 'fixture: the release put the app\'s container in a slice, so nothing below was tested').to.match(/\/system\.slice\/docker-[0-9a-f]{64}\.scope$/); + expect(runningBefore, 'the app\'s container was not running after the upgrade, so there was nothing to kill').to.not.be.empty; + }); + + it('held every read of docker\'s records of its containers, and went out of service', function () { + expect(readsWereHeld, 'a read of docker\'s container records returned, so the disk was never held').to.equal(true); + expect(trippedAt, 'the node never went out of service for the stall, so nothing below was tested').to.not.equal(null); + }); + + it('kills the app\'s container though neither docker\'s records nor the database name the app', function () { + expect(stopped, 'the node never said it had stopped its apps').to.not.equal(null); + expect(stopped.apps.map((a) => a.app), 'fixture: the node found the app by name, so nothing below was tested').to.not.include(appName); + expect(runnableAfterWindow, 'the app\'s container can still run after the stop window').to.deep.equal([]); + }); + + it('says nothing of the containers outside the app slices can run', function () { + expect(stopped?.outsideSlices, 'the node did not report the containers outside the app slices').to.not.equal(undefined); + expect(stopped.outsideSlices.containers, 'the node killed no container outside the app slices').to.be.at.least(1); + expect(stopped.outsideSlices.runnable, 'the node reports them as ones it cannot tell can still run, or names their processes').to.deep.equal([]); + }); +}); diff --git a/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js b/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js new file mode 100644 index 0000000000..f45a625fc1 --- /dev/null +++ b/test-infra/runner/tests/3319-a-boot-does-not-wait-for-a-host-stall-state-write.js @@ -0,0 +1,72 @@ +import { describe, it, before, after } from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { execInContainer, crashFluxos, releaseFluxos } from '../framework/container.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A FluxOS that starts while the directory its host-stall state is kept in takes no +// writes: the state there cannot be read, so the start's first reading writes it, +// and that write does not end. The start returns once the reading has acted, and +// boot goes on to its next step. +// +// The node's /run/fluxos is a filesystem of its own, frozen, so a read of it answers +// and a write to it is held in the kernel. + +const STATE_DIR = '/run/fluxos'; +const STATE_FILE = `${STATE_DIR}/host-stall.json`; +const IMAGE = '/tmp/flux-e2e-run-fluxos.img'; +// Boot's line after the host-stall service has started (serviceManager). +const NEXT_STEP = 'MongoDB connected'; + +describe('a boot does not wait for a host stall\'s state write', function () { + let env; + dumpLogsOnFailure(() => env); + let client; + let writesHeld; + let bootLinesBefore; + + const sh = async (command) => { + const r = await execInContainer(client.container, command); + if (r.exitCode !== 0) throw new Error(`fixture: ${command}: ${r.output}`); + return r.stdout; + }; + + before(async function () { + this.timeout(300000); + env = await createTestEnv({ hookCtx: this, nodes: 1, tickerAutostart: false }); + [client] = env.clients; + await waitFor(() => env.nodeLogCount(0, NEXT_STEP) >= 1, { timeout: 120000, interval: 1000, label: 'the node booted' }); + + await crashFluxos(client.container, { hold: true }); + await sh(`rm -rf ${STATE_DIR} && mkdir -p ${STATE_DIR} && dd if=/dev/zero of=${IMAGE} bs=1M count=16 status=none` + + ` && mke2fs -q -t ext4 ${IMAGE} && mount -o loop ${IMAGE} ${STATE_DIR}` + + ` && printf '{ not json' > ${STATE_FILE} && sync -f ${STATE_FILE} && fsfreeze -f ${STATE_DIR}`); + // a write held in the kernel cannot be killed: it runs in the background, and is + // held while its marker has not appeared + const marker = `/tmp/state-write-returned-${Date.now()}`; + const probe = await execInContainer(client.container, + `setsid sh -c 'echo x > ${STATE_DIR}/probe && touch ${marker}' /dev/null 2>&1 & sleep 3; test ! -e ${marker}`); + writesHeld = probe.exitCode === 0; + + bootLinesBefore = env.nodeLogCount(0, NEXT_STEP); + await releaseFluxos(client.container); + }); + + after(async function () { + this.timeout(60000); + if (client) await execInContainer(client.container, `fsfreeze -u ${STATE_DIR}; umount -l ${STATE_DIR}; rm -f ${IMAGE}`).catch(() => {}); + await env?.teardown(); + }); + + it('holds every write to the state directory', function () { + expect(writesHeld, 'a write to the state directory returned, so nothing below was tested').to.equal(true); + }); + + it('goes on to the next step of boot while the state write does not end', async function () { + this.timeout(120000); + await waitFor(() => env.nodeLogCount(0, NEXT_STEP) > bootLinesBefore, { + timeout: 60000, interval: 1000, label: 'boot waited for the host stall\'s state write', + }); + }); +}); diff --git a/test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js b/test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js new file mode 100644 index 0000000000..fd9df47cb4 --- /dev/null +++ b/test-infra/runner/tests/3320-a-device-is-not-removed-from-a-folder-a-revert-holds.js @@ -0,0 +1,125 @@ +// weight: heavy +/* + * Syncthing drops a deleted device from every folder that lists it, outside + * FluxOS's writes to those folders. A revert of a standby's changes holds the + * folder against those writes from the moment it finds the folder the one it + * checked until the revert is sent, so a device the monitor no longer wants + * stays configured while any folder lists it, and leaves the folder through the + * folder's own write. + * + * The primary's device becomes one the monitor deletes while the standby's revert + * of a change to the primary's file is held after its last read: the primary + * stops announcing, its location expires, and another device is configured under + * its name. The standby's change is reverted with the primary still in the + * folder, so it is not deleted, and the primary leaves the folder only once the + * revert has been sent. + * + * A = the standby, B = the primary, C = holds nothing. + */ +import { + describe, it, before, after, afterEach, +} from 'mocha'; +import { expect } from 'chai'; +import { + placeSilentPrimaryApp, A, B, C, +} from '../framework/silent-primary.js'; +import { execInContainer, crashFluxos } from '../framework/container.js'; +import { dbClient } from '../framework/db-client.js'; +import { + readPath, scanFolder, getFolderConfig, getFolderStatus, getDeviceId, getConfiguredDevices, addDevice, +} from '../framework/syncthing-real.js'; +import { waitFor } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +const SIGTERM_EXPIRY_S = 120; +const REVERT_SENT = 'syncthing:beforeLocalChangeRevertSent'; +const REVERT_MS = 5 * 60 * 1000; + +function step(what) { + console.log(`# 3320 ${new Date().toISOString()} ${what}`); +} + +describe('a device is not removed from a folder a revert holds', function () { + let fleet; + dumpLogsOnFailure(() => fleet?.env); + const appName = `e2eheldrevert${Date.now()}`; + const folder = `flux${appName}_${appName}`; + const root = `/mnt/appdata/flux-apps/${folder}`; + const client = (i) => fleet.env.clients[i]; + const sh = async (i, command) => { + const r = await execInContainer(client(i).container, command); + if (r.exitCode !== 0) throw new Error(`${command}: ${r.output}`); + return r.stdout; + }; + const folderDevices = async () => ((await getFolderConfig(client(A), folder))?.devices ?? []).map((d) => d.deviceID); + + before(async function () { + this.timeout(900000); + fleet = await placeSilentPrimaryApp({ hookCtx: this, appName, sigtermExpiryS: SIGTERM_EXPIRY_S }); + }); + + afterEach(async function () { + await client(A)?.releaseAllCheckpoints().catch(() => {}); + }); + + after(async function () { + this.timeout(60000); + await client(A)?.releaseAllCheckpoints().catch(() => {}); + fleet?.stop(); + await fleet?.env?.teardown(); + }); + + it('reverts the standby\'s change while the primary is still in the folder, and removes the primary from it after', async function () { + this.timeout(REVERT_MS + 600000); + const held = `${root}/appdata/held`; + await sh(B, `printf primary > ${held}`); + await scanFolder(client(B), folder); + await waitFor(async () => (await readPath(client(A), held)) === 'primary', { + timeout: 120000, interval: 2000, label: 'the primary\'s file reached the standby', + }); + const primaryDevice = await getDeviceId(client(B)); + const primaryName = (await getConfiguredDevices(client(A))).find((d) => d.deviceID === primaryDevice)?.name; + expect(primaryName, 'fixture: the standby has no device configured for the primary').to.be.a('string'); + + await client(A).holdCheckpoint(REVERT_SENT, folder); + const from = client(A).getLastEventId(); + step('the standby changes the primary\'s file, and its revert is held after its last read'); + await sh(A, `printf standby > ${held}`); + await scanFolder(client(A), folder); + await client(A).waitForEvent('checkpoint:held', (d) => d.name === REVERT_SENT && d.key === folder, REVERT_MS, { afterId: from }); + + step('the primary stops announcing, its location expires, and another device is configured under its name'); + await crashFluxos(client(B).container, { hold: true }); + const expired = await dbClient(A + 1).expireAppLocation(appName, client(B).ip); + expect(expired, 'fixture: the standby held no location for the primary, so none expired').to.be.at.least(1); + await addDevice(client(A), { + deviceID: await getDeviceId(client(C)), + name: primaryName, + address: `tcp://${client(B).ip}:16129`, + }); + const passFrom = client(A).getLastEventId(); + await client(A).waitForEvent('syncthing:passComplete', () => true, 120000, { afterId: passFrom }); + const configuredWhileHeld = (await getConfiguredDevices(client(A))).some((d) => d.deviceID === primaryDevice); + const inFolderWhileHeld = (await folderDevices()).includes(primaryDevice); + + const releasedAt = client(A).getLastEventId(); + await client(A).releaseCheckpoint(REVERT_SENT, folder); + await client(A).waitForEvent('syncthing:localChangesReverted', (d) => d.folder === folder, REVERT_MS, { afterId: releasedAt }) + .catch(() => { + throw new Error('the standby did not revert its change once the revert was released'); + }); + await waitFor(async () => !(await folderDevices()).includes(primaryDevice), { + timeout: 300000, interval: 2000, label: 'the primary left the standby\'s folder', + }); + await waitFor(async () => (await getFolderStatus(client(A), folder))?.state === 'idle', { + timeout: 120000, interval: 2000, label: 'the standby\'s folder is idle', + }); + + // Reverted while the primary was in the folder, the file is the primary's + // version once pulled, or the standby's while the primary cannot be reached; + // reverted with no other device in the folder, it is deleted. + expect(await readPath(client(A), held), 'the standby\'s file after its revert').to.be.oneOf(['primary', 'standby']); + expect(configuredWhileHeld, 'the primary\'s device was deleted while a folder listed it').to.equal(true); + expect(inFolderWhileHeld, 'the primary left the folder while its revert was held').to.equal(true); + }); +}); diff --git a/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js new file mode 100644 index 0000000000..fb9a7e518c --- /dev/null +++ b/test-infra/runner/tests/3501-an-announcement-is-judged-one-entry-at-a-time.js @@ -0,0 +1,224 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST, getSubnetConfig } from '../framework/subnet-config.js'; +import { assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { execInContainer } from '../framework/container.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A version 2 apprunning message names every app its sender holds. Each entry is +// judged on its own: one without a name and a hash is refused and the rest of the +// message is stored and relayed as if it were not there. Each node says what it +// did with a message in one network:apprunning event: the entries it wrote, the +// names it refused, whether it cleared the sender's rows, and whether it relayed. +// +// The stub peer is a listed node with its own key. Nodes 1 and 2 cannot reach it, +// so they hear what it says only through node 0's relay, and a message node 0 +// does not store goes no further. +// +// The last case is a real node that holds an app installed without a hash beside +// one installed normally: it leaves the first out of its announcements +// (app:running.unannounced), and every node goes on storing them. + +const REAL_NODES = 3; +const STUB_INDEX = 3; +const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); +const BEHIND_RELAY = [1, 2]; +// A node names itself in its messages by its socket address, a stub by its IP. +const bare = (address) => String(address).replace(/:16127$/, ''); + +function appComponent(appName) { + return { + name: 'alpha', + description: 'one entry at a time', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +describe('an announcement is judged one entry at a time', function () { + let env; + dumpLogsOnFailure(() => env); + let stub; + const stamp = Date.now(); + const stubIp = () => getSubnetConfig().nodeIp(STUB_INDEX + 1); + const rowNamesAt = async (i, address) => (await dbClient(i + 1).getAppLocationsByIp(address)).map((row) => row.name); + const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); + // What each node did with the message from `ip` that names `name`, after `afterIds`. + const handledBy = (nodes, afterIds, ip, name, timeout = 60000) => Promise.all(nodes.map(async (i) => (await env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(ip) && [...(d.apps ?? []).map((a) => a.name), ...(d.refused ?? [])].includes(name), + timeout, + { afterId: afterIds[i] }, + )).data)); + const everyNodeStored = async (afterIds, ip, name) => { + const handled = await handledBy(realNodes, afterIds, ip, name); + handled.forEach((d, i) => expect(d.apps.map((a) => a.name), `node ${i} did not store ${name}`).to.include(name)); + return handled; + }; + const message = (apps) => ({ + type: 'fluxapprunning', version: 2, apps, ip: stubIp(), broadcastedAt: Date.now(), osUptime: 10000, staticIp: false, + }); + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ + hookCtx: this, + nodes: REAL_NODES + 1, + stubPeers: [STUB_INDEX], + stubPeeredWith: { [STUB_INDEX]: [0] }, + tickerAutostart: false, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, + }); + stub = env.stubPeerClients.get(STUB_INDEX); + // Nodes 1 and 2 cannot reach the stub, cut off before the fleet peers so + // neither ever holds a connection to it. + await Promise.all(BEHIND_RELAY.map(async (i) => { + const cut = await execInContainer(env.clients[i].container, + `iptables -I INPUT -s ${stubIp()} -j DROP && iptables -I OUTPUT -d ${stubIp()} -j DROP`); + if (cut.exitCode !== 0) throw new Error(`fixture: node ${i} could not cut the stub off (exit ${cut.exitCode}): ${cut.output}`); + })); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + const peeredWithStub = await Promise.all(BEHIND_RELAY.map(async (i) => { + const [outgoing, incoming] = await Promise.all([env.clients[i].getPeers(), env.clients[i].getIncomingPeers()]); + return JSON.stringify([...(outgoing.data ?? []), ...(incoming.data ?? [])]).includes(stubIp()); + })); + expect(peeredWithStub, 'fixture: a node behind the relay is peered with the stub').to.deep.equal([false, false]); + expect(await stub.connectedNodes(), 'fixture: the stub holds connections beyond node 0').to.equal(1); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('stores and relays the entries with a name and hash, and refuses the one without', async function () { + this.timeout(120000); + const kept = `e2eentrykept${stamp}`; + const refused = `e2eentryrefused${stamp}`; + const afterIds = marks(); + + const sent = await stub.broadcast(message([ + { name: kept, hash: `hash-${kept}`, runningSince: new Date().toISOString() }, + { name: refused, runningSince: new Date().toISOString() }, + ])); + expect(sent.sent, 'fixture: the message reached node 0 and no other').to.equal(1); + + const handled = await everyNodeStored(afterIds, stubIp(), kept); + handled.forEach((d, i) => { + expect(d.refused, `node ${i} did not refuse the entry without a hash`).to.deep.equal([refused]); + expect(d.apps.map((a) => a.name), `node ${i} wrote the entry without a hash`).to.not.include(refused); + }); + expect(handled[0].relayed, 'node 0 did not relay the message to the nodes behind it').to.equal(true); + // eslint-disable-next-line no-restricted-syntax + for (const i of realNodes) { + // eslint-disable-next-line no-await-in-loop + expect(await rowNamesAt(i, stubIp()), `node ${i} holds a row for the entry without a hash`).to.not.include(refused); + } + }); + + // An empty v2 message deletes the sender's rows. A message whose entries were + // all refused named apps: it stores nothing, goes no further, and is never + // read as one that named none. + it('stores and relays nothing of a message whose every entry is refused, and keeps the sender\'s rows', async function () { + this.timeout(180000); + const held = `e2eentryheld${stamp}`; + const refusedOnly = `e2eentryrefusedonly${stamp}`; + const marker = `e2eentrymarker${stamp}`; + let afterIds = marks(); + await stub.runApp(held, { hash: `hash-${held}` }); + await everyNodeStored(afterIds, stubIp(), held); + + afterIds = marks(); + await stub.broadcast(message([{ name: refusedOnly }])); + // A node handles one peer's messages in the order they were sent, so a node + // that stored this has handled the refused message before it. + await stub.runApp(marker, { hash: `hash-${marker}` }); + await everyNodeStored(afterIds, stubIp(), marker); + + const [onNodeZero] = await handledBy([0], afterIds, stubIp(), refusedOnly); + expect(onNodeZero).to.deep.include({ apps: [], refused: [refusedOnly], cleared: false, relayed: false }); + // Relayed, it would have reached them before the marker did. + await Promise.all(BEHIND_RELAY.map((i) => assertNoEvent(env.clients[i], 'network:apprunning', + (d) => d.refused?.includes(refusedOnly), 0, { afterId: afterIds[i] }))); + // eslint-disable-next-line no-restricted-syntax + for (const i of realNodes) { + // eslint-disable-next-line no-await-in-loop + expect(await rowNamesAt(i, stubIp()), `node ${i} deleted the sender's rows`).to.include(held); + } + }); + + // What the case above guards against is real: a message naming no apps does + // delete the sender's rows, on node 0 and on the nodes behind it. + it('deletes the sender\'s rows on a message that names no apps', async function () { + this.timeout(120000); + const held = `e2eentrycleared${stamp}`; + let afterIds = marks(); + await stub.runApp(held, { hash: `hash-${held}` }); + await everyNodeStored(afterIds, stubIp(), held); + + afterIds = marks(); + await stub.broadcast(message([])); + await Promise.all(realNodes.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', (d) => bare(d.ip) === bare(stubIp()) && d.cleared === true, 60000, { afterId: afterIds[i] }, + ))); + // eslint-disable-next-line no-restricted-syntax + for (const i of realNodes) { + // eslint-disable-next-line no-await-in-loop + expect(await rowNamesAt(i, stubIp()), `node ${i} kept the sender's rows`).to.deep.equal([]); + } + }); + + it('leaves out an app installed without a hash, and every node goes on storing the rest', async function () { + this.timeout(420000); + const appName = `e2eentryreal${stamp}`; + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ name: appName, instances: 1, compose: [appComponent(appName)] }); + // Node 1: it announces on its own interval. Node 2's ring neighbour is the + // stub's slot, which it cannot reach, so it holds fewer peers than the + // threshold that starts a node's announcements. + const idx = 1; + await installOnNodes(env, app, [idx]); + const holder = socketAddr(idx + 1); + const others = realNodes.filter((i) => i !== idx); + + // An app installed without a hash, as a node carries one it took before it + // upgraded. Everything else about it is the real app's. + const unhashed = `e2eentryunhashed${stamp}`; + const { hash, ...withoutHash } = await dbClient(idx + 1).localApp(appName); + expect(hash, 'fixture: the real app is installed with a hash').to.be.a('string'); + const afterIds = marks(); + await dbClient(idx + 1).seedLocalApp({ ...withoutHash, name: unhashed }); + + const { data: announced } = await env.clients[idx].waitForEvent( + 'app:running', (d) => d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[idx] }, + ); + expect(await dbClient(idx + 1).localApp(unhashed), 'fixture: the unhashed app was still installed when the holder announced').to.not.equal(null); + expect(announced.apps.map((a) => a.name), 'the holder announced the app it has no hash for').to.not.include(unhashed); + expect(announced.unannounced, 'the holder did not say what it left out').to.deep.equal([unhashed]); + + // Every other node stores the announcements made since. + await Promise.all(others.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[i] }, + ).catch(() => { throw new Error(`node ${i} stored no announcement from the holder since the unhashed app was installed`); }))); + await Promise.all(others.map((i) => assertNoEvent(env.clients[i], 'network:apprunning', + (d) => bare(d.ip) === bare(holder) && d.refused?.length > 0, 0, { afterId: afterIds[i] }))); + }); +}); diff --git a/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js new file mode 100644 index 0000000000..fa147252db --- /dev/null +++ b/test-infra/runner/tests/3502-a-node-keeps-its-own-record-of-when-it-took-an-app.js @@ -0,0 +1,189 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { waitForBootSettled, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { restartFluxos } from '../framework/container.js'; +import { socketAddr } from '../framework/state-events.js'; +import { fluxTeamKey } from '../framework/keys.js'; +import { authenticate } from '../auth.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node records when it took an app (app:placementRecorded) and announces that +// as the app's runningSince. The record follows the app's data on the node: a +// hard install, which creates the app's volume, writes it; a removal, which +// deletes the volume, deletes it (app:placementRemoved); a soft redeploy keeps +// the volume and the record. The network's copy of the node's own location row +// is not where it comes from, because that row lapses while the app goes on +// running. At boot, records of apps not installed are removed +// (app:placementsTidied) and an installed app without one is dated at the +// node's own row (app:placementsBackfilled). + +const REAL_NODES = 3; +const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); +const DAY_MS = 24 * 60 * 60 * 1000; +const bare = (address) => String(address).replace(/:16127$/, ''); + +describe('a node keeps its own record of when it took an app', function () { + let env; + dumpLogsOnFailure(() => env); + let idx; + let holder; + let others; + // What the holder's announcements carry for the app, from the first one on. + let announcedSince; + const appName = `e2eplacement${Date.now()}`; + + const runningSinceIn = (data) => { + const entry = data.apps?.find((a) => a.name === appName); + return entry ? new Date(entry.runningSince).getTime() : null; + }; + // The runningSince every other node stored from the holder's next announcement + // after `afterIds`. + async function nextStoredRunningSince(afterIds) { + const events = await Promise.all(others.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), + 120000, + { afterId: afterIds[i] }, + ))); + return events.map(({ data }) => runningSinceIn(data)); + } + const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); + const teamAuth = async () => (await authenticate(env.clients[idx].url, fluxTeamKey())).zelidauth; + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ hookCtx: this, nodes: REAL_NODES, tickerAutostart: false }); + await bootAndPeer(env); + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ + name: appName, + instances: 1, + compose: [{ + name: 'alpha', + description: 'placement record', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + idx = await seedAndInstall(env, app); + holder = socketAddr(idx + 1); + others = realNodes.filter((i) => i !== idx); + const { data: first } = await env.clients[idx].waitForEvent('app:running', (d) => runningSinceIn(d) !== null, 120000); + announcedSince = runningSinceIn(first); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('records when it took the app at install, and announces that', async function () { + this.timeout(180000); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000); + expect(recorded.source).to.equal('install'); + expect(new Date(recorded.placedAt).getTime(), 'the announced runningSince is not the recorded one').to.equal(announcedSince); + + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'another node stored a different runningSince').to.deep.equal(others.map(() => announcedSince)); + }); + + it('keeps announcing the same time after its own location rows lapse', async function () { + this.timeout(180000); + const afterIds = marks(); + // Every node's row for the holder, the holder's own included, as after an + // expiry, an eviction or a sigterm. + await Promise.all(realNodes.map((i) => dbClient(i + 1).deleteAppLocation(appName, holder))); + + const stored = await nextStoredRunningSince(afterIds); + expect(stored, 'the holder re-dated the app when its row lapsed').to.deep.equal(others.map(() => announcedSince)); + }); + + it('keeps the record, and the time it announces, through a soft redeploy', async function () { + this.timeout(300000); + const afterIds = marks(); + const body = await env.clients[idx].redeployApp(appName, await teamAuth(), { force: false }); + const { data: redeployed } = await env.clients[idx].waitForEvent('app:softRedeployed', (d) => d.name === appName, 240000, { afterId: afterIds[idx] }); + expect(redeployed.outcome, `the redeploy did not reinstall the app: ${body.slice(-400)}`).to.equal('installed'); + + await assertNoEvent(env.clients[idx], 'app:placementRemoved', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[idx] }); + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'the soft redeploy re-dated the app').to.deep.equal(others.map(() => announcedSince)); + }); + + // A hard redeploy starts the app again from an empty volume, so it starts a + // new record: the copy holds none of the data the old date stood for. + it('starts a new record, and announces it, after a hard redeploy', async function () { + this.timeout(300000); + const afterIds = marks(); + const body = await env.clients[idx].redeployApp(appName, await teamAuth(), { force: true }); + const { data: rebuilt } = await env.clients[idx].waitForEvent('app:rebuilt', (d) => d.name === appName, 240000, { afterId: afterIds[idx] }); + expect(rebuilt.outcome, `the redeploy did not reinstall the app: ${body.slice(-400)}`).to.equal('installed'); + + await env.clients[idx].waitForEvent('app:placementRemoved', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); + expect(recorded.source).to.equal('install'); + const placedAt = new Date(recorded.placedAt).getTime(); + expect(placedAt, 'the new record is not newer than the old one').to.be.above(announcedSince); + + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'the announced runningSince is not the new record').to.deep.equal(others.map(() => placedAt)); + announcedSince = placedAt; + }); + + // A node that took an app before it had these records gets one at boot, dated + // at the runningSince its own row already carries; a record for an app it does + // not hold is dropped. + it('makes its records match its installed apps at boot, dating a missing one at its own row', async function () { + this.timeout(420000); + const rowSince = Date.now() - 3 * DAY_MS; + const ghost = `e2eplacementghost${Date.now()}`; + await dbClient(idx + 1).deletePlacement(appName); + await dbClient(idx + 1).seedPlacement(ghost, Date.now() - DAY_MS); + await dbClient(idx + 1).setAppLocationRunningSince(appName, holder, rowSince); + + // FluxOS alone, so the downtime is seconds and the node keeps its apps. + const afterIds = marks(); + await restartFluxos(env.clients[idx].container); + await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); + + const { data: tidied } = await env.clients[idx].waitForEvent('app:placementsTidied', () => true, 10000, { afterId: afterIds[idx] }); + expect(tidied, 'the boot did not remove the record of the app it does not hold').to.deep.equal({ removed: 1 }); + const { data: backfilled } = await env.clients[idx].waitForEvent('app:placementsBackfilled', () => true, 10000, { afterId: afterIds[idx] }); + expect(backfilled, 'the boot did not give the app a record').to.deep.equal({ created: 1, fromLocationRows: 1 }); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 10000, { afterId: afterIds[idx] }); + expect(recorded.source).to.equal('locationRow'); + expect(new Date(recorded.placedAt).getTime(), 'the missing record was not dated at the node\'s own row').to.equal(rowSince); + + const stored = await nextStoredRunningSince(marks()); + expect(stored, 'the announced runningSince is not the recorded one').to.deep.equal(others.map(() => rowSince)); + }); + + it('drops the record with the app', async function () { + this.timeout(240000); + const afterIds = marks(); + const body = await env.clients[idx].removeAppLocally(appName, await teamAuth()); + await env.clients[idx].waitForEvent('app:removed', (d) => d.name === appName, 180000, { afterId: afterIds[idx] }) + .catch(() => { throw new Error(`the app was not removed: ${body.slice(-400)}`); }); + + await env.clients[idx].waitForEvent('app:placementRemoved', (d) => d.name === appName, 30000, { afterId: afterIds[idx] }); + }); +}); diff --git a/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js new file mode 100644 index 0000000000..115476721e --- /dev/null +++ b/test-infra/runner/tests/3503-who-may-place-or-test-an-app-by-hand.js @@ -0,0 +1,118 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { buildAppSpec, registerApp } from '../framework/app-helper.js'; +import { + appOwnerKey, fluxTeamKey, nodeKey, userKey, +} from '../framework/keys.js'; +import { authenticate } from '../auth.js'; +import { waitFor, waitForBootSettled, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// An app registered and not yet paid for exists only as a temporary message. +// installapplocally places registered apps, for the flux team, and never one of +// these. testappinstall tries one out - installs it, checks it starts, tears it +// down, and never announces it - for its owner or the flux team, and refuses any +// other caller before a container is made. + +const REAL_NODES = 3; +const NODE = 0; + +describe('who may place or test an app by hand', function () { + let env; + dumpLogsOnFailure(() => env); + let appName; + let messageHash; + const client = () => env.clients[NODE]; + const authAs = async (key) => (await authenticate(client().url, key)).zelidauth; + + const marks = () => env.clients.map((c) => c.getLastEventId()); + // installapplocally placed nothing: no record of taking the app was written, + // and the app is not installed. + async function assertNotPlaced(afterIds) { + await assertNoEvent(client(), 'app:placementRecorded', (d) => d.name === appName, 0, { afterId: afterIds[NODE] }); + expect(await dbClient(NODE + 1).localApp(appName), 'the app is installed on the node').to.equal(null); + } + // A test install leaves nothing of the app behind. + const assertTornDown = () => waitFor(async () => (await dbClient(NODE + 1).localApp(appName)) === null, { + timeout: 60000, interval: 2000, label: 'the test install is torn down', + }); + + before(async function () { + this.timeout(420000); + // Registration wants peers both ways; a fleet this small holds one each way. + env = await createTestEnv({ + hookCtx: this, + nodes: REAL_NODES, + tickerAutostart: false, + configOverrides: { fluxapps: { minOutgoing: 1, minIncoming: 1 } }, + }); + await bootAndPeer(env, { minOutbound: 1, minInbound: 1 }); + await Promise.all(env.clients.map((c) => waitForBootSettled(c, 240000))); + await waitFor(async () => { + const [outgoing, incoming] = await Promise.all([client().getPeers(), client().getIncomingPeers()]); + return (outgoing.data?.length ?? 0) >= 1 && (incoming.data?.length ?? 0) >= 1; + }, { timeout: 120000, interval: 2000, label: 'the registering node has a peer each way' }); + + appName = `e2ehandplaced${Date.now()}`; + const spec = buildAppSpec({ name: appName }); + const result = await registerApp(client().url, nodeKey(1), spec); + expect(result.status, `fixture: the registration was refused: ${JSON.stringify(result)}`).to.equal('success'); + messageHash = result.data; + await waitFor(async () => (await client().getTempMessages(messageHash)).data?.length > 0, { + timeout: 60000, interval: 2000, label: 'the node holds the temporary message', + }); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('refuses its owner placing it with installapplocally', async function () { + this.timeout(120000); + const afterIds = marks(); + const body = await client().installAppLocally(messageHash, await authAs(appOwnerKey())); + + expect(body, 'the owner was not refused').to.match(/Unauthorized/); + await assertNotPlaced(afterIds); + }); + + it('does not place it for the flux team either', async function () { + this.timeout(120000); + const afterIds = marks(); + const body = await client().installAppLocally(messageHash, await authAs(fluxTeamKey())); + + expect(body, 'installapplocally read the temporary message').to.match(new RegExp(`Application Specifications of ${messageHash} not found`)); + await assertNotPlaced(afterIds); + }); + + it('refuses another user testing it', async function () { + this.timeout(120000); + const body = await client().testAppInstall(messageHash, await authAs(userKey())); + + expect(body, 'another user was not refused').to.match(/Unauthorized/); + await assertTornDown(); + }); + + it('lets its owner test it, and leaves nothing behind', async function () { + this.timeout(300000); + const body = await client().testAppInstall(messageHash, await authAs(appOwnerKey())); + + expect(body, 'the owner\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); + await assertTornDown(); + }); + + it('lets the flux team test it', async function () { + this.timeout(300000); + const body = await client().testAppInstall(messageHash, await authAs(fluxTeamKey())); + + expect(body, 'the flux team\'s test did not install the app').to.match(new RegExp(`Flux App ${appName} successfully installed and launched`)); + await assertTornDown(); + }); +}); diff --git a/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js b/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js new file mode 100644 index 0000000000..5f63653ed5 --- /dev/null +++ b/test-infra/runner/tests/3504-a-synced-announcement-is-judged-one-entry-at-a-time.js @@ -0,0 +1,120 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { dbClient } from '../framework/db-client.js'; +import { apprunningEvent, socketAddr } from '../framework/state-events.js'; +import { bootAndPeer } from '../framework/reconciler-suite.js'; +import { waitForDaemonReady, waitForNodeStatus, waitForOrchestratorState } from '../framework/wait.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node joining the network learns where apps run from its peers' event logs, +// and judges each announcement there one entry at a time, as it judges one that +// arrives by gossip: an entry without a name and a hash is refused +// (network:apprunningRefused, path sync) and the rest are stored. +// +// A peer that has not upgraded serves whatever it last recorded for a node, +// which may be an announcement whose every entry the joiner refuses. The joiner +// here is a node coming back: it already holds a row for that node and the +// readable announcement behind it. It prunes a node's rows only against an +// announcement it can store, so the refused one deletes neither the row nor the +// readable announcement in the log it serves its own peers. + +const RUNNING = [0, 1, 2, 3]; +const JOINER = 4; +// The nodes the announcements are about, by node number: real nodes holding no +// apps, so the seeded announcements are the only ones there are about them. +const MIXED_NODE = 3; +const REFUSED_NODE = 4; + +describe('a synced announcement is judged one entry at a time', function () { + let env; + dumpLogsOnFailure(() => env); + let joiner; + let refused; + const stamp = Date.now(); + const kept = `e2esynckept${stamp}`; + const dropped = `e2esyncdropped${stamp}`; + const held = `e2esyncheld${stamp}`; + const probe = `e2esyncprobe${stamp}`; + const rowNames = async (nodeNum) => (await dbClient(JOINER + 1).getAppLocationsByIp(socketAddr(nodeNum))).map((row) => row.name); + + before(async function () { + this.timeout(600000); + env = await createTestEnv({ + hookCtx: this, + nodes: RUNNING.length + 1, + deferredNodes: 1, + tickerAutostart: false, + // The joiner learns from these, and a node answers a state sync only once + // its own has finished. + syncedNodes: RUNNING, + }); + await bootAndPeer(env); + + // Booted cut off, so the seeded announcements' acceptance window spans the + // sync and not the boot. + await env.holdOutPendingNode(JOINER, RUNNING); + joiner = await env.startNode(JOINER); + await waitForDaemonReady(joiner); + await waitForNodeStatus(joiner, (d) => d.confirmed === true, 30000); + + const at = Date.now(); + // What the joiner already holds about REFUSED_NODE from before it went away: + // the readable announcement and the row it wrote. + await dbClient(JOINER + 1).seedAppStateEvent(await apprunningEvent({ nodeNum: REFUSED_NODE, broadcastedAt: at - 5000, apps: [held] })); + await dbClient(JOINER + 1).seedAppLocation({ + name: held, ip: socketAddr(REFUSED_NODE), hash: `hash-${held}`, broadcastedAt: at - 5000, runningSince: at - 5000, + }); + // What every peer serves: for MIXED_NODE one readable entry and one without a + // hash; for REFUSED_NODE a newer announcement whose only entry has no hash, + // the one a peer that has not upgraded holds for a node it last heard that from. + const events = [ + await apprunningEvent({ + nodeNum: MIXED_NODE, + broadcastedAt: at, + apps: [kept, { name: dropped, runningSince: new Date(at).toISOString() }], + }), + await apprunningEvent({ + nodeNum: REFUSED_NODE, broadcastedAt: at, apps: [{ name: probe, runningSince: new Date(at).toISOString() }], + }), + ]; + await Promise.all(RUNNING.map((i) => dbClient(i + 1).seedAppStateEvents(events.map((event) => ({ ...event }))))); + + const mark = joiner.getLastEventId(); + await env.releasePendingNode(JOINER, RUNNING); + await env.startDiscovery([JOINER]); + await waitForOrchestratorState(joiner, 'READY', 180000); + // Every peer's response stored and pruned. + await joiner.waitForEvent('ephemeralSync:allComplete', () => true, 180000, { afterId: mark }); + refused = joiner.getEventBuffer() + .filter((e) => e.id > mark && e.event === 'network:apprunningRefused' && e.data.path === 'sync') + .flatMap((e) => e.data.refusals); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('stores the entries with a name and hash, and refuses the one without', async function () { + const names = await rowNames(MIXED_NODE); + expect(names).to.include(kept); + expect(names, 'the joiner stored an entry without a hash').to.not.include(dropped); + expect(refused.some((r) => r.ip === socketAddr(MIXED_NODE) && r.names.includes(dropped)), 'the joiner did not report the entry it refused').to.equal(true); + }); + + it('keeps the row it holds for a node past a newer announcement whose every entry it refuses', async function () { + const names = await rowNames(REFUSED_NODE); + expect(names, 'the refused announcement pruned the rows the readable one stood behind').to.include(held); + expect(names, 'the joiner stored an entry without a hash').to.not.include(probe); + expect(refused.some((r) => r.ip === socketAddr(REFUSED_NODE) && r.names.includes(probe)), 'the joiner did not report the announcement it refused').to.equal(true); + }); + + it('keeps the readable announcement in the log it serves its own peers', async function () { + const logged = await dbClient(JOINER + 1).getAppStateEvents({ ip: socketAddr(REFUSED_NODE), type: 'apprunning' }); + expect(logged.map((event) => event.data.apps.map((a) => a.name)), 'the refused announcement took the readable one\'s place').to.deep.equal([[held]]); + }); +}); diff --git a/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js b/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js new file mode 100644 index 0000000000..d3a391ae29 --- /dev/null +++ b/test-infra/runner/tests/3505-an-unhashed-app-across-a-rolling-upgrade.js @@ -0,0 +1,123 @@ +// base-image: FLUX_E2E_BASE_TAG +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { bootAndPeer, installOnNodes } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A network part way through an upgrade. Node 0 runs the FluxOS the others +// upgrade from (FLUX_E2E_BASE_TAG), which refuses a whole apprunning message for +// one entry without a hash; nodes 1 and 2 run this build. +// +// An upgraded node holding an app installed without a hash leaves it out of its +// announcements (app:running.unannounced), so the node not yet upgraded goes on +// storing the rest (network:apprunning). A node not yet upgraded announces such +// an app, and the upgraded nodes write its other entries and refuse that one, +// both of which their network:apprunning says. +// +// Without FLUX_E2E_BASE_TAG the suite fails before booting, naming what it needs. + +const BASE_TAG = process.env.FLUX_E2E_BASE_TAG; +const NOT_UPGRADED = 0; +const UPGRADED = [1, 2]; +const bare = (address) => String(address).replace(/:16127$/, ''); + +function appComponent(appName) { + return { + name: 'alpha', + description: 'rolling upgrade', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }; +} + +describe('an app installed without a hash, across a rolling upgrade', function () { + let env; + dumpLogsOnFailure(() => env); + const stamp = Date.now(); + + // Install `appName` on `holder`, then give it an app installed without a hash. + async function holderWithUnhashedApp(appName, holder) { + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ name: appName, instances: 1, compose: [appComponent(appName)] }); + await installOnNodes(env, app, [holder]); + const { hash, ...withoutHash } = await dbClient(holder + 1).localApp(appName); + expect(hash, 'fixture: the app is installed with a hash').to.be.a('string'); + const unhashed = `${appName}unhashed`; + const afterIds = env.clients.map((c) => c.getLastEventId()); + await dbClient(holder + 1).seedLocalApp({ ...withoutHash, name: unhashed }); + return { unhashed, afterIds }; + } + const storedFrom = (i, afterId, holder, appName) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(socketAddr(holder + 1)) && d.apps?.some((a) => a.name === appName), + 120000, + { afterId }, + ); + + before(async function () { + this.timeout(420000); + if (!BASE_TAG) { + throw new Error('FLUX_E2E_BASE_TAG names the node image of the FluxOS the nodes upgrade from: build it from that tree with FLUX_E2E_TAG= ./test-infra/build-images.sh fluxos-01'); + } + env = await createTestEnv({ + hookCtx: this, + nodes: 3, + tickerAutostart: false, + nodeImageTags: { [NOT_UPGRADED]: BASE_TAG }, + }); + if (BASE_TAG === env.nodeImageTag(UPGRADED[0])) throw new Error('FLUX_E2E_BASE_TAG must name another build than FLUX_E2E_TAG'); + await bootAndPeer(env); + }); + + after(async function () { + this.timeout(60000); + await env?.teardown(); + }); + + it('an upgraded node leaves the app out, and the node not yet upgraded goes on storing the rest', async function () { + this.timeout(420000); + const appName = `e2erollupgraded${stamp}`; + const holder = UPGRADED[0]; + const { unhashed, afterIds } = await holderWithUnhashedApp(appName, holder); + + const { data: announced } = await env.clients[holder].waitForEvent( + 'app:running', (d) => d.apps?.some((a) => a.name === appName), 120000, { afterId: afterIds[holder] }, + ); + expect(announced.unannounced, 'the upgraded holder announced the app it has no hash for').to.deep.equal([unhashed]); + await storedFrom(NOT_UPGRADED, afterIds[NOT_UPGRADED], holder, appName) + .catch(() => { throw new Error('the node not yet upgraded stored no announcement from the upgraded holder since the unhashed app was installed'); }); + }); + + it('a node not yet upgraded announces the app, and the upgraded nodes store the rest and refuse it', async function () { + this.timeout(420000); + const appName = `e2erollbase${stamp}`; + const holder = NOT_UPGRADED; + const { unhashed, afterIds } = await holderWithUnhashedApp(appName, holder); + + await Promise.all(UPGRADED.map(async (i) => { + const { data: stored } = await storedFrom(i, afterIds[i], holder, appName) + .catch(() => { throw new Error(`upgraded node ${i} stored no announcement from the node not yet upgraded since the unhashed app was installed`); }); + expect(stored.apps.map((a) => a.name)).to.include(appName); + expect(stored.refused, `upgraded node ${i} did not refuse the app without a hash`).to.deep.equal([unhashed]); + expect(await dbClient(i + 1).appLocationAt(unhashed, socketAddr(holder + 1)), `upgraded node ${i} holds a row for the app without a hash`).to.equal(null); + })); + }); +}); diff --git a/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js b/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js new file mode 100644 index 0000000000..5c1b038ee6 --- /dev/null +++ b/test-infra/runner/tests/3506-a-node-that-boots-without-its-address-dates-its-records-once-it-has-one.js @@ -0,0 +1,128 @@ +// weight: medium +import { + describe, it, before, after, +} from 'mocha'; +import { expect } from 'chai'; +import { createTestEnv } from '../framework/test-env.js'; +import { pushImage } from '../framework/registry-helper.js'; +import { buildSeedableApp } from '../framework/seed-helper.js'; +import { REGISTRY_REPO_HOST } from '../framework/subnet-config.js'; +import { waitForBootSettled, assertNoEvent } from '../framework/wait.js'; +import { bootAndPeer, seedAndInstall } from '../framework/reconciler-suite.js'; +import { dbClient } from '../framework/db-client.js'; +import { restartFluxos } from '../framework/container.js'; +import { setBenchmarkUnresolved } from '../framework/daemon-control.js'; +import { socketAddr } from '../framework/state-events.js'; +import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; + +// A node's location rows are filed under its address, and an installed app +// without a placement record is dated from this node's own row. A node that +// boots before benchmark reports its address writes no record until it has one +// (app:placementRecorded), then dates the app at its own row, and its next +// announcement carries that date to its peers. + +const REAL_NODES = 3; +const realNodes = Array.from({ length: REAL_NODES }, (_unused, i) => i); +const DAY_MS = 24 * 60 * 60 * 1000; +// Production's location lifetime, so this node's own row still stands while it +// has no address. +const LOCATION_TTL_S = 30 * 60; +// Long enough that the node is not out of service while the case holds its +// address back, at the harness's 15 s ask interval. +const PUBLIC_IP_ASKS_BEFORE_DOS = 40; +// A node syncs its apps only once it can be heard, which needs its address, and +// removes them when it has not synced within this: long enough to hold the +// address back and keep the app. +const BOOT_SYNC_TIMEOUT_MS = 300000; +// One harness ask interval and three confirmation polls: a backfill that ran +// without the address has written by then. +const UNRESOLVED_WINDOW_MS = 20000; +const bare = (address) => String(address).replace(/:16127$/, ''); + +describe('a node that boots without its address dates its records once it has one', function () { + let env; + dumpLogsOnFailure(() => env); + let idx; + let holder; + let others; + const appName = `e2enoaddress${Date.now()}`; + + const marks = () => realNodes.map((i) => env.clients[i].getLastEventId()); + const runningSinceIn = (data) => { + const entry = data.apps?.find((a) => a.name === appName); + return entry ? new Date(entry.runningSince).getTime() : null; + }; + + before(async function () { + this.timeout(420000); + env = await createTestEnv({ + hookCtx: this, + nodes: REAL_NODES, + tickerAutostart: false, + configOverrides: { + fluxapps: { locationTtlS: LOCATION_TTL_S, publicIpAsksBeforeDos: PUBLIC_IP_ASKS_BEFORE_DOS }, + system: { bootSyncTimeoutMs: BOOT_SYNC_TIMEOUT_MS }, + }, + }); + await bootAndPeer(env); + await pushImage(appName, 'v1'); + const app = await buildSeedableApp({ + name: appName, + instances: 1, + compose: [{ + name: 'alpha', + description: 'no address at boot', + repotag: `${REGISTRY_REPO_HOST}/${appName}:v1`, + ports: [], + domains: [''], + environmentParameters: [], + commands: [], + containerPorts: [80], + containerData: '/tmp', + cpu: 0.1, + ram: 100, + hdd: 1, + repoauth: '', + }], + }); + idx = await seedAndInstall(env, app); + holder = socketAddr(idx + 1); + others = realNodes.filter((i) => i !== idx); + await env.clients[idx].waitForEvent('app:running', (d) => runningSinceIn(d) !== null, 120000); + }); + + after(async function () { + this.timeout(60000); + if (holder) await setBenchmarkUnresolved(holder, false).catch(() => {}); + await env?.teardown(); + }); + + it('writes no record while it has no address, then dates the app at its own row and announces that', async function () { + this.timeout(480000); + const rowSince = Date.now() - 3 * DAY_MS; + await dbClient(idx + 1).deletePlacement(appName); + await dbClient(idx + 1).setAppLocationRunningSince(appName, holder, rowSince); + expect((await dbClient(idx + 1).appLocationAt(appName, holder))?.runningSince?.getTime(), 'fixture: the node\'s own row').to.equal(rowSince); + + await setBenchmarkUnresolved(holder, true); + const afterIds = marks(); + await restartFluxos(env.clients[idx].container); + + await assertNoEvent(env.clients[idx], 'app:placementRecorded', (d) => d.name === appName, UNRESOLVED_WINDOW_MS, { afterId: afterIds[idx] }); + + const resolvedIds = marks(); + await setBenchmarkUnresolved(holder, false); + await waitForBootSettled(env.clients[idx], 240000, { afterId: afterIds[idx] }); + const { data: recorded } = await env.clients[idx].waitForEvent('app:placementRecorded', (d) => d.name === appName, 120000, { afterId: resolvedIds[idx] }); + expect(recorded.source, 'the record was not dated from the node\'s own row').to.equal('locationRow'); + expect(new Date(recorded.placedAt).getTime(), 'the record is not the node\'s own row\'s date').to.equal(rowSince); + + const stored = await Promise.all(others.map((i) => env.clients[i].waitForEvent( + 'network:apprunning', + (d) => bare(d.ip) === bare(holder) && d.apps?.some((a) => a.name === appName), + 120000, + { afterId: resolvedIds[i] }, + ))); + expect(stored.map(({ data }) => runningSinceIn(data)), 'a peer stored a different runningSince').to.deep.equal(others.map(() => rowSince)); + }); +}); diff --git a/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js b/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js index 27e4230505..280ebc8dfd 100644 --- a/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js +++ b/test-infra/runner/tests/46-reconciler-boot-broadcast-ordering.js @@ -83,7 +83,7 @@ describe('first post-boot broadcast: complete, never empty, never destructive', // buffer holds every apprunning it received across the reboot window) const received = peerClient.getEventBuffer().filter((e) => e.id > afterId && e.event === 'network:apprunning'); expect(received.length, 'buffer must hold the apprunning waitForEvent just matched (filter shape drift)').to.be.greaterThan(0); - const empty = received.filter((e) => Array.isArray(e.data?.apps) && e.data.apps.length === 0); + const empty = received.filter((e) => e.data?.cleared === true); expect(empty, 'no empty fluxapprunning v2 snapshots').to.have.lengthOf(0); // (a): the peer's location rows for the app survived the whole sequence diff --git a/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js b/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js index 11f8d66c6e..87c1965556 100644 --- a/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js +++ b/test-infra/runner/tests/51-reconciler-syncthing-coldstart.js @@ -5,7 +5,7 @@ import { getAppContainerStatus, blockPeerAccess, unblockPeerAccess, execInContainer, } from '../framework/container.js'; import { - setSyncState, setNoPeerData, resetSyncState, + resetSyncState, pinColdStart, pinHolding, } from '../framework/syncthing-control.js'; import { electMaster, resetFdm } from '../framework/fdm-control.js'; import { getSubnetConfig } from '../framework/subnet-config.js'; @@ -34,6 +34,13 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; const subnet = getSubnetConfig(); +// The state machine's checkpoint before it measures what this node holds of a +// receive-only folder, keyed by folder id. +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; +// Passes over a folder a node is counted deciding while another holder is held: +// more than the leadership confirmation needs, so a win would have been acted on. +const HELD_PASSES = 4; + async function isUp(client, appName) { const status = await getAppContainerStatus(client.container, appName); return !!(status && status.status.startsWith('Up')); @@ -44,45 +51,6 @@ async function countUp(env, indices, appName) { return ups.filter(Boolean).length; } -// pin a folder to a true cold-start shape on every holder: empty global of its own -// and no connected peer holding the data (set BEFORE install so the first election -// evaluation sees it, not the stub's default phantom-synced peer) -async function pinColdStart(holders, folder) { - await Promise.all(holders.map((i) => Promise.all([ - setSyncState({ - ip: subnet.nodeIp(i + 1), folder, state: 'idle', globalBytes: 0, inSyncBytes: 0, - // Honest only for a bare g:/r: app. A component declaring an f:/m:/ml: mount has - // FluxOS scaffolding on its volume that a real daemon counts here, and the stub - // cannot see a disk - so those shapes belong on syncthing: 'binary', not here. - receiveOnlyChangedFiles: 0, - }), - setNoPeerData({ ip: subnet.nodeIp(i + 1), folder }), - ]))); -} - -// One holder carries the owner's data, declared the way seedSyncScopedData declares it: -// the ENTRIES, not a count, because localHoldings totals the entry list and excludes the -// scaffolding in it by type and by name. Written after pinColdStart, which it overwrites -// for this one node. -async function pinHolding(index, folder, bytes) { - const modified = new Date().toISOString(); - await setSyncState({ - ip: subnet.nodeIp(index + 1), - folder, - state: 'idle', - globalBytes: 0, - inSyncBytes: 0, - localChanged: [ - { - name: 'appdata', type: 'FILE_INFO_TYPE_DIRECTORY', size: 128, deleted: false, modified, - }, - { - name: 'appdata/seed-data', type: 'FILE_INFO_TYPE_FILE', size: bytes, deleted: false, modified, - }, - ], - }); - await setNoPeerData({ ip: subnet.nodeIp(index + 1), folder }); -} describe('reconciler cold start - fresh multi-node placement, no seeded source', function () { let env; @@ -91,6 +59,7 @@ describe('reconciler cold start - fresh multi-node placement, no seeded source', const gApp = `e2ecoldg${Date.now()}`; const claimApp = `e2ecoldclaim${Date.now()}`; const splitApp = `e2ecoldsplit${Date.now()}`; + const waitApp = `e2ecoldwait${Date.now()}`; const holders = [0, 1, 2]; // Asked rather than assumed. `holders[0]` is the lowest address only while the // list happens to be written in order, and it silently names the wrong node @@ -165,6 +134,51 @@ describe('reconciler cold start - fresh multi-node placement, no seeded source', // signed endpoint and ranks the field on the answers. A unit test reaches the // comparator with a claims object handed to it; only a fleet reaches it through a // node's own reading of its volume and a peer's reading of that node. + // THE CLAIM HAS TO HAVE BEEN MADE. A holder answers what it holds once its monitor + // has measured the folder; between its install and that pass it has measured + // nothing, and read then it holds nothing, so the field falls to the address order + // over the node holding the owner's data. The third holder - neither the + // address-order winner nor the one holding the data - is held just before it + // measures the folder, which is the window an install opens on every holder. + it('r: nobody seeds while a holder has not yet measured what it holds, and then the holder seeds', async function () { + this.timeout(300000); + const folder = `flux${waitApp}_${waitApp}`; + const unmeasured = holders.find((i) => i !== seedIndex && i !== claimHolder); + expect(unmeasured, 'no holder is left between the address-order winner and the data holder').to.not.equal(undefined); + const passes = () => env.clients[seedIndex].getDecisionCount('syncthing:folderPass', folder, 'evaluated'); + + await pushImage(waitApp, 'v1'); + const spec = await buildSeedableSyncthingApp({ name: waitApp, mode: 'r' }); + await pinColdStart(holders, folder); + await pinHolding(claimHolder, folder, claimBytes); + await env.clients[unmeasured].holdCheckpoint(BEFORE_HOLDINGS, folder); + const from = env.clients[unmeasured].getLastEventId(); + try { + await installOnNodes(env, spec, holders); + await env.clients[unmeasured].waitForEvent('checkpoint:held', (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 120000, { afterId: from }); + + // The address-order winner decides the folder pass after pass while the third + // holder has not measured it - long enough to confirm a win it would act on. + const passesFrom = await passes(); + await waitFor(async () => (await passes()) >= passesFrom + HELD_PASSES, { + timeout: 120000, interval: 2000, label: `the address-order winner ran ${HELD_PASSES} passes over the folder`, + }); + expect( + await countUp(env, holders, waitApp), + 'a holder seeded while another had not yet measured what it holds', + ).to.equal(0); + } finally { + await env.clients[unmeasured].releaseCheckpoint(BEFORE_HOLDINGS, folder); + } + + await waitFor(async () => (await countUp(env, holders, waitApp)) >= 1, { + timeout: 90000, interval: 3000, label: 'a holder seeds once every holder has measured what it holds', + }); + await new Promise((r) => { setTimeout(r, 12000); }); + expect(await countUp(env, holders, waitApp)).to.equal(1); + expect(await isUp(env.clients[claimHolder], waitApp), 'the node holding the data did not seed').to.equal(true); + }); + it('r: the holder seeds, against the address the field would otherwise elect', async function () { this.timeout(180000); // The premise, asserted so a fixture that stops being divergent fails here rather diff --git a/test-infra/runner/tests/53-terminal-exec-crash-safety.js b/test-infra/runner/tests/53-terminal-exec-crash-safety.js index ffc289620a..862435b505 100644 --- a/test-infra/runner/tests/53-terminal-exec-crash-safety.js +++ b/test-infra/runner/tests/53-terminal-exec-crash-safety.js @@ -4,7 +4,7 @@ import { io } from 'socket.io-client'; import { createTestEnv } from '../framework/test-env.js'; import { authenticate } from '../auth.js'; import { appOwnerKey } from '../framework/keys.js'; -import { execInContainer, getAppContainerStatus } from '../framework/container.js'; +import { execInContainer, getAppContainerStatus, fluxosPid } from '../framework/container.js'; import { waitFor } from '../framework/wait.js'; import { bootAndPeer, seedSimpleApp } from '../framework/reconciler-suite.js'; import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; @@ -28,17 +28,10 @@ import { dumpLogsOnFailure } from '../framework/log-on-failure.js'; // // The assertion that matters in both cases is the same: a socket error reaches the // client AND the FluxOS process is the same one it was before (an exit(1) is -// invisible from outside once the watchdog respawns it, so we compare the pid). +// invisible from outside once systemd starts it again, so we compare the pid). const TERMINAL_TIMEOUT_MS = 20000; -// The node app.js child pid, written by the image entrypoint watchdog. It changes -// if and only if FluxOS died and was respawned. -async function fluxosPid(container) { - const { stdout } = await execInContainer(container, 'cat /tmp/fluxos.pid 2>/dev/null || echo ""'); - return stdout.trim(); -} - async function apiAlive(client) { const res = await client.get('/flux/version').catch(() => null); return !!res; @@ -119,7 +112,7 @@ describe('docker terminal fails cleanly and never crashes the node', function () this.timeout(180000); const pidBefore = await fluxosPid(client.container); - expect(pidBefore).to.not.equal(''); + expect(pidBefore).to.not.equal(null); // A real session against the harness stub image, which has no shell and no // passwd: exec the one binary it carries (/bin/pause) as the image-default @@ -179,7 +172,7 @@ describe('docker terminal fails cleanly and never crashes the node', function () }, { timeout: 60000, interval: 2000, label: 'app container stopped' }); const pidBefore = await fluxosPid(client.container); - expect(pidBefore).to.not.equal(''); + expect(pidBefore).to.not.equal(null); const result = await openTerminal(client, zelidauth, identifier); diff --git a/test-infra/runner/tests/68-masterslave-election-order.js b/test-infra/runner/tests/68-masterslave-election-order.js index 090ecc352c..dcb1c7d9fc 100644 --- a/test-infra/runner/tests/68-masterslave-election-order.js +++ b/test-infra/runner/tests/68-masterslave-election-order.js @@ -52,6 +52,10 @@ const BEFORE_START = 'masterSlave:beforeStart'; // writing it, keyed by folder id. const BEFORE_FOLDER_WRITE = 'syncthing:beforeFolderWrite'; +// The syncthing monitor's checkpoint before a receiving folder's holdings are +// measured, keyed by folder id. +const BEFORE_HOLDINGS = 'syncthing:beforeHoldings'; + async function isUp(client, appName) { const status = await getAppContainerStatus(client.container, appName); return !!(status && status.status.startsWith('Up')); @@ -651,9 +655,35 @@ describe('primary election under a divergent placement order', function () { // // The seed is placed first, so the genesis winner is election index 0 and both // standbys are below it: each has a turn to book. + // + // The seed is held before it measures the folder until every later holder has, + // so it cannot win on its own before the others are placed: the seed is decided + // over the whole field, and the address order decides it because no holder + // holds anything. const deployedFrom = holders.map((i) => env.clients[i].getLastEventId()); - await deploy(lossApp, placementOrderWithSeedAt(holders, 0)); const folder = folderOf(lossApp); + const others = holders.filter((i) => i !== seedIndex); + const waitedOnSeed = (i) => env.clients[i].getDecisionCount('syncthing:seedDecision', folder, 'holderNotLookedAt'); + await env.clients[seedIndex].holdCheckpoint(BEFORE_HOLDINGS, folder); + try { + await deploy(lossApp, placementOrderWithSeedAt(holders, 0)); + await env.clients[seedIndex].waitForEvent('checkpoint:held', + (d) => d.name === BEFORE_HOLDINGS && d.key === folder, 120000, { afterId: deployedFrom[holders.indexOf(seedIndex)] }); + // A holder that has measured the folder waits on the seed, which has not. + await Promise.all(others.map((i) => waitFor(async () => (await waitedOnSeed(i)) > 0, { + timeout: 120000, interval: 1000, label: `holder ${i} measured the folder and waits on the seed`, + }))); + } finally { + await env.clients[seedIndex].releaseCheckpoint(BEFORE_HOLDINGS, folder) + .catch((err) => console.warn(`cleanup: checkpoint release failed: ${err.message}`)); + } + // The genesis winner is the holder the seed election picks. A folder pinned + // synced before it has picked lets the primary election start index 0 instead. + const seeded = async () => (await Promise.all(holders.map((i) => env.clients[i] + .getDecisionCount('syncthing:seedDecision', folder, 'seeded')))).reduce((sum, n) => sum + n, 0); + await waitFor(async () => (await seeded()) > 0, { + timeout: 120000, interval: 1000, label: 'a holder decided to seed the folder', + }); await Promise.all(holders.map((i) => setSynced({ ip: subnet.nodeIp(i + 1), folder }))); await Promise.all(holders.map((i, k) => env.clients[i].waitForEvent('syncthing:folderReady', (d) => d.folder === folder, 180000, { afterId: deployedFrom[k] }))); diff --git a/test-infra/syncthing-stub/index.js b/test-infra/syncthing-stub/index.js index 05a9bebfb1..b5ef894b6d 100644 --- a/test-infra/syncthing-stub/index.js +++ b/test-infra/syncthing-stub/index.js @@ -759,8 +759,12 @@ app.get('/rest/db/localchanged', (req, res) => { if (ov?.statusUnreadable) return res.status(500).json({ error: 'simulated unreadable folder status' }); const folderCfg = reqState(req).folders.get(req.query.folder || ''); const isReceiveOnly = !folderCfg || folderCfg.type === 'receiveonly'; - const files = isReceiveOnly && Array.isArray(ov?.localChanged) ? ov.localChanged : []; - return res.json({ files, page: 1, perpage: 65536 }); + const all = isReceiveOnly && Array.isArray(ov?.localChanged) ? ov.localChanged : []; + // Paged as syncthing pages it: `page` is 1-based, `perpage` defaults to 65536. + const page = Math.max(1, Number(req.query.page) || 1); + const perpage = Math.max(1, Number(req.query.perpage) || 65536); + const files = all.slice((page - 1) * perpage, page * perpage); + return res.json({ files, page, perpage }); }); app.get('/rest/db/status', (req, res) => { diff --git a/test-infra/systemd/dockerd.service b/test-infra/systemd/dockerd.service index 1786443716..23d258ce1b 100644 --- a/test-infra/systemd/dockerd.service +++ b/test-infra/systemd/dockerd.service @@ -1,15 +1,21 @@ [Unit] -Description=Inner Docker daemon (harness node, systemd mode) +Description=Inner Docker daemon (harness node) After=network.target +# Restarted as often as a suite asks: a test that restarts it repeatedly is not a +# unit failing. +StartLimitIntervalSec=0 [Service] ExecStartPre=/bin/rm -f /var/run/docker.pid -# No flags: /etc/docker/daemon.json (written by the entrypoint's systemd -# branch) owns the configuration — dockerd refuses the same directive from +# No flags: /etc/docker/daemon.json (written by the entrypoint) owns the +# configuration — dockerd refuses the same directive from # both a flag and the file. ExecStart=/usr/bin/dockerd Restart=always RestartSec=1 +# The node's output stream, where the harness reads the unit's lines (entrypoint.sh). +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit Delegate=yes LimitNOFILE=1048576 diff --git a/test-infra/systemd/flux-node-network.service b/test-infra/systemd/flux-node-network.service new file mode 100644 index 0000000000..b652973207 --- /dev/null +++ b/test-infra/systemd/flux-node-network.service @@ -0,0 +1,20 @@ +[Unit] +Description=Node network shape and global IPv6 (harness node) +# Built once dockerd answers and before FluxOS starts: a node reads its network at +# boot, and dockerd refuses to start under the routes a shape adds. Wanted, not +# required: a dockerd restart leaves this unit and FluxOS running, as on a node. +After=dockerd.service +Wants=dockerd.service +Before=fluxos.service pm2-fluxuser.service + +[Service] +Type=oneshot +RemainAfterExit=yes +EnvironmentFile=/etc/fluxos-harness.env +ExecStartPre=/bin/bash -c 'for i in $$(seq 1 30); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1' +ExecStart=/flux/test-infra/node-network.sh +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit + +[Install] +WantedBy=multi-user.target diff --git a/test-infra/systemd/flux-pm2-logs.service b/test-infra/systemd/flux-pm2-logs.service new file mode 100644 index 0000000000..99141371b0 --- /dev/null +++ b/test-infra/systemd/flux-pm2-logs.service @@ -0,0 +1,19 @@ +[Unit] +# pm2 writes FluxOS's output to its own log files; this streams them into the +# node's output, where the harness reads FluxOS's lines. +Description=pm2 log stream (harness node, legacy multitool install) +After=pm2-fluxuser.service +BindsTo=pm2-fluxuser.service + +[Service] +User=fluxuser +EnvironmentFile=/etc/fluxos-harness.env +Environment=PM2_HOME=/home/fluxuser/.pm2 +ExecStart=/usr/bin/env pm2 logs --raw --lines 0 +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit +Restart=always +RestartSec=1 + +[Install] +WantedBy=multi-user.target diff --git a/test-infra/systemd/fluxos.service b/test-infra/systemd/fluxos.service index 36a5fb37a2..1d182ec570 100644 --- a/test-infra/systemd/fluxos.service +++ b/test-infra/systemd/fluxos.service @@ -1,24 +1,31 @@ [Unit] # The unit carries the production unit's name. -Description=FluxOS (harness node, systemd mode) -After=dockerd.service +Description=FluxOS (harness node) +After=dockerd.service flux-node-network.service Wants=dockerd.service +Requires=flux-node-network.service +# Restarted as often as a suite asks: a test that restarts it repeatedly is not a +# unit failing. +StartLimitIntervalSec=0 [Service] # User= is what makes systemd set HOME, and a legacy node finds its install at -# $HOME/zelflux (the image links /root/zelflux to /flux). +# $HOME/zelflux (the image links /root/zelflux and /home/fluxuser/zelflux to +# /flux). An unprivileged node replaces it with a drop-in (entrypoint.sh). User=root WorkingDirectory=/flux # Container env does not cross into systemd services (the manager # environment arrives empty) — the entrypoint dumps it here before # exec'ing systemd. EnvironmentFile=/etc/fluxos-harness.env -# Mirror the legacy entrypoint's gate: FluxOS boots only once the inner -# dockerd answers. +# FluxOS boots only once the inner dockerd answers. ExecStartPre=/bin/bash -c 'for i in $$(seq 1 30); do docker info >/dev/null 2>&1 && exit 0; sleep 1; done; echo "dockerd not ready" >&2; exit 1' ExecStart=/usr/bin/node app.js Restart=always RestartSec=1 +# The node's output stream, where the harness reads the unit's lines (entrypoint.sh). +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit [Install] WantedBy=multi-user.target diff --git a/test-infra/systemd/pm2-fluxuser.service b/test-infra/systemd/pm2-fluxuser.service new file mode 100644 index 0000000000..11bb87d5e9 --- /dev/null +++ b/test-infra/systemd/pm2-fluxuser.service @@ -0,0 +1,25 @@ +[Unit] +# A legacy node the multitool installed, as `pm2 startup` registers it: pm2 runs as +# the operator's account and starts FluxOS through start.sh, which runs npm start - +# npm install, then FluxOS - from the account's home, with the flags the multitool +# gives it. The kill timeout is the fleet's to choose; unset, pm2 uses its own default. +Description=PM2 process manager +After=dockerd.service flux-node-network.service +Wants=dockerd.service +Requires=flux-node-network.service + +[Service] +Type=forking +User=fluxuser +EnvironmentFile=/etc/fluxos-harness.env +Environment=PM2_HOME=/home/fluxuser/.pm2 +# There is no registry here, so npm answers its install from the image's own. +Environment=NPM_CONFIG_OFFLINE=true NPM_CONFIG_AUDIT=false NPM_CONFIG_FUND=false NPM_CONFIG_UPDATE_NOTIFIER=false +PIDFile=/home/fluxuser/.pm2/pm2.pid +ExecStart=/bin/bash -c 'cd "$$HOME" && exec pm2 start "$$HOME/zelflux/start.sh" --name flux --max-memory-restart 1500M --restart-delay 30000 --max-restarts 40 --time $${FLUX_PM2_KILL_TIMEOUT_MS:+--kill-timeout "$$FLUX_PM2_KILL_TIMEOUT_MS"}' +ExecStop=/usr/bin/env pm2 kill +StandardOutput=append:/var/lib/flux-harness/stream +StandardError=inherit + +[Install] +WantedBy=multi-user.target diff --git a/test-infra/systemd/syncthing-forward.service b/test-infra/systemd/syncthing-forward.service index 0fd517fec1..d487a170ed 100644 --- a/test-infra/systemd/syncthing-forward.service +++ b/test-infra/systemd/syncthing-forward.service @@ -1,11 +1,10 @@ [Unit] -Description=Syncthing stub port forward (apiport+2, harness node, systemd mode) +Description=Syncthing stub port forward (apiport+2, harness node) [Service] EnvironmentFile=/etc/fluxos-harness.env # $$ keeps the shell's own expansion out of systemd's specifier/variable -# expansion; the arithmetic and defaults run in bash, exactly like the -# legacy entrypoint's forward. +# expansion; the arithmetic and defaults run in bash. ExecStart=/bin/bash -c 'exec socat TCP-LISTEN:$$(( $${FLUX_API_PORT:-16127} + 2 )),fork,reuseaddr TCP:$${FLUX_SYNCTHING_HOST}:$${FLUX_SYNCTHING_PORT:-8384}' Restart=always RestartSec=1 diff --git a/test-infra/write-node-config.sh b/test-infra/write-node-config.sh new file mode 100755 index 0000000000..52e0df8727 --- /dev/null +++ b/test-infra/write-node-config.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# Writes the node's FluxOS config overlay, ZelBack/config/local.js: the per-node file +# NODE_CONFIG_DIR names, with the runner's overrides (FLUX_TEST_CONFIG, JSON) merged +# over it. Run by the entrypoint at boot and by an in-place upgrade, which takes the +# new build's per-node file the same way. +# +# app.js hardcodes NODE_CONFIG_DIR to ZelBack/config/ (cannot be overridden from env +# - fluxbenchd hashes that directory for tamper detection), so the test config is +# overlaid into it. +set -e + +if [ -n "$NODE_CONFIG_DIR" ] && [ -d "$NODE_CONFIG_DIR" ]; then + cp "$NODE_CONFIG_DIR"/default.js /flux/ZelBack/config/local.js + cp "$(dirname "$NODE_CONFIG_DIR")/shared.js" /flux/ZelBack/ 2>/dev/null || true +fi + +# Merged OVER the per-node file, which is where the per-node database names come +# from - replacing that file rather than merging would take them with it. A file +# rather than NODE_CONFIG: the config package merges that variable over every file +# whatever directory is pinned, so it could redirect any endpoint without touching +# the directory fluxbenchd hashes, and the entry points delete it. +if [ -n "$FLUX_TEST_CONFIG" ]; then + node -e ' + const fs = require("fs"); + const target = "/flux/ZelBack/config/local.js"; + const base = fs.existsSync(target) ? require(target) : {}; + const isPlain = (v) => v && typeof v === "object" && !Array.isArray(v); + const merge = (a, b) => { + const out = { ...a }; + for (const [k, v] of Object.entries(b)) out[k] = isPlain(v) && isPlain(a[k]) ? merge(a[k], v) : v; + return out; + }; + const merged = merge(base, JSON.parse(process.env.FLUX_TEST_CONFIG)); + fs.writeFileSync(target, `module.exports = ${JSON.stringify(merged, null, 2)};\n`); + ' +fi diff --git a/tests/unit/advancedWorkflows.test.js b/tests/unit/advancedWorkflows.test.js index cca81e9cb7..349a15dbda 100644 --- a/tests/unit/advancedWorkflows.test.js +++ b/tests/unit/advancedWorkflows.test.js @@ -13,6 +13,8 @@ const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileg const dbHelper = require('../../ZelBack/src/services/dbHelper'); const appsRuntimeState = require('../../ZelBack/src/services/appManagement/appsRuntimeState'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); +const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutcome'); +const { MongoNetworkTimeoutError } = require('mongodb'); const log = require('../../ZelBack/src/lib/log'); const https = require('https'); const syncthingService = require('../../ZelBack/src/services/syncthingService'); @@ -22,14 +24,21 @@ const primaryRoleChanges = require('../../ZelBack/src/services/appLifecycle/prim const { appsFolder } = require('../../ZelBack/src/services/utils/appConstants'); const syncthingFolderStateMachine = require('../../ZelBack/src/services/appMonitoring/syncthingFolderStateMachine'); const { RECONNECT_GRACE_MS } = require('../../ZelBack/src/services/appMonitoring/peerFolderLiveness'); +const { PeerComponent } = require('../../ZelBack/src/services/appMonitoring/peerComponent'); const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); const { addressMoveEvents, EVENTS: ADDRESS_MOVE_EVENTS } = require('../../ZelBack/src/services/utils/addressMoveEvents'); const appInstaller = require('../../ZelBack/src/services/appLifecycle/appInstaller'); +const appPlacements = require('../../ZelBack/src/services/appManagement/appPlacements'); +const appRebuild = require('../../ZelBack/src/services/appLifecycle/appRebuild'); const appNetworkLinker = require('../../ZelBack/src/services/appLifecycle/appNetworkLinker'); const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); const appInspector = require('../../ZelBack/src/services/appManagement/appInspector'); const volumeService = require('../../ZelBack/src/services/utils/volumeService'); const dockerServiceModule = require('../../ZelBack/src/services/dockerService'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); +const appQueryService = require('../../ZelBack/src/services/appQuery/appQueryService'); +const appUninstaller = require('../../ZelBack/src/services/appLifecycle/appUninstaller'); // The operator stop lock as appsRuntimeState.operatorStopState answers it. const UNLOCKED = { stopped: false, startRequested: false, force: false }; @@ -98,6 +107,50 @@ describe('advancedWorkflows tests', () => { expect(globalState.installationInProgress, 'the node is left holding an install that never began').to.be.false; }); + it('stops without tearing anything down when the database does not answer', async () => { + // eslint-disable-next-line global-require + const appUninstaller = require('../../ZelBack/src/services/appLifecycle/appUninstaller'); + sinon.stub(generalService, 'nodeTier').resolves('cumulus'); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + + const result = await advancedWorkflows.softRegisterAppLocally({ name: 'testapp', version: 3 }, false, makeRes()); + + expect(result).to.equal(InstallOutcome.DEFERRED); + expect(removeAppLocally.called, 'a soft registration destroyed the data it exists to keep').to.be.false; + expect(globalState.installationInProgress).to.be.false; + }); + + // A soft registration keeps the app's volumes, so a folder that already has a + // state keeps it; one the spec adds has not been looked at. + it('records a folder the spec adds as not looked at, and keeps the state of one it already has', async () => { + sinon.stub(generalService, 'nodeTier').resolves('cumulus'); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + const insert = sinon.stub(dbHelper, 'insertOneToDatabase').resolves({ insertedId: 'id' }); + sinon.stub(appNetworkLinker, 'checkAppNetworkRequirements').resolves(); + sinon.stub(appInstaller, 'ensureAppDockerNetwork').resolves(); + sinon.stub(appInstaller, 'installApplicationSoft').rejects(new Error('Error pulling image')); + sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + globalState.folderStates.clear(); + globalState.folderStates.set('fluxkept_softapp', { state: 'measured', bytes: 10, newestModified: 1 }); + + await advancedWorkflows.softRegisterAppLocally({ + name: 'softapp', + version: 4, + compose: [ + { name: 'kept', containerData: 'g:/data', cpu: 0.5, ram: 500, hdd: 5 }, + { name: 'added', containerData: 'g:/data', cpu: 0.5, ram: 500, hdd: 5 }, + ], + }, false, makeRes()); + + sinon.assert.called(insert); + expect(globalState.folderStates.get('fluxkept_softapp')).to.deep.equal({ state: 'measured', bytes: 10, newestModified: 1 }); + expect(globalState.folderStates.get('fluxadded_softapp')).to.deep.equal({ state: 'notLookedAt' }); + globalState.folderStates.clear(); + }); + it('does not release a hold it never took', async () => { // Refused because another install owns the node. Lowering the flag here // hands the node to the next caller while that install is still running. @@ -110,6 +163,19 @@ describe('advancedWorkflows tests', () => { expect(nodeTier.called, 'the refusal is decided before any work is done').to.be.false; expect(globalState.installationInProgress, 'a refusal released someone else\'s hold').to.be.true; }); + + it('refuses an app whose removal is pending', async () => { + const nodeTier = sinon.stub(generalService, 'nodeTier').resolves('cumulus'); + globalState.departingApps.enter(appSpec.name); + try { + const result = await advancedWorkflows.softRegisterAppLocally(appSpec, false, makeRes()); + + expect(result).to.equal(InstallOutcome.BUSY); + expect(nodeTier.called, 'the refusal is decided before any work is done').to.be.false; + } finally { + globalState.departingApps.leave(appSpec.name); + } + }); }); describe('setRemovalInProgress and getRemovalInProgress tests', () => { @@ -297,6 +363,83 @@ describe('advancedWorkflows tests', () => { }); }); + describe('forceAppRemovals', () => { + let remove; + + beforeEach(() => { + resetGlobalState(); + sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('203.0.113.60:16127'); + sinon.stub(appQueryService, 'listAllApps').resolves({ status: 'success', data: [{ Names: ['/fluxweb_orphanapp'] }] }); + sinon.stub(appQueryService, 'installedApps').resolves({ status: 'success', data: [] }); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(registryManager, 'getApplicationGlobalSpecifications').resolves({ name: 'orphanapp' }); + sinon.stub(serviceHelper, 'delay').resolves(); + remove = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + }); + + afterEach(() => sinon.restore()); + + it('broadcasts the removal of an orphan whose location cannot be read', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').rejects(new MongoNetworkTimeoutError('timed out')); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.calledOnceWithExactly(remove, 'orphanapp', null, true, true, true); + }); + + it('does not broadcast the removal of an orphan this node is not listed for', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.calledOnceWithExactly(remove, 'orphanapp', null, true, true, false); + }); + + it('removes nothing when the installed apps cannot be read', async () => { + appQueryService.installedApps.resolves({ status: 'error', data: { message: 'timed out' } }); + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + const warned = sinon.stub(log, 'warn'); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.notCalled(remove); + sinon.assert.calledWithMatch(warned, /Unable to read the apps docker runs and the installed apps/); + }); + + it('removes nothing when the apps docker runs cannot be read', async () => { + appQueryService.listAllApps.resolves({ status: 'error', data: { message: 'docker did not answer' } }); + sinon.stub(dbHelper, 'findOneInDatabase').resolves(null); + const warned = sinon.stub(log, 'warn'); + + await advancedWorkflows.forceAppRemovals(); + + sinon.assert.notCalled(remove); + sinon.assert.calledWithMatch(warned, /Unable to read the apps docker runs and the installed apps/); + }); + }); + + describe('checkAndRemoveEnterpriseAppsOnNonArcane', () => { + afterEach(() => sinon.restore()); + + it('runs again once the registry it could not read answers, and not after', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findInDatabase').resolves([{ name: 'enterpriseCandidate' }]); + const registryRead = sinon.stub(dbHelper, 'findOneInDatabase'); + registryRead.onFirstCall().rejects(new MongoNetworkTimeoutError('timed out')); + registryRead.resolves({ name: 'enterpriseCandidate', version: 7 }); + + await advancedWorkflows.checkAndRemoveEnterpriseAppsOnNonArcane(); + sinon.assert.calledOnce(registryRead); + + await clock.tickAsync(5 * 60 * 1000); + sinon.assert.calledTwice(registryRead); + + await clock.tickAsync(60 * 60 * 1000); + sinon.assert.calledTwice(registryRead); + }); + }); + describe('softRedeployComponent tests', () => { let globalState; let res; @@ -323,6 +466,17 @@ describe('advancedWorkflows tests', () => { resetGlobalState(); }); + it('answers the database error, not "not found", when the registry cannot be read, and clears its flag', async () => { + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findOneInDatabase').rejects(new MongoNetworkTimeoutError('timed out')); + + const error = await advancedWorkflows.softRedeployComponent('myapp', 'frontend', res).catch((caught) => caught); + + expect(error).to.be.instanceOf(MongoNetworkTimeoutError); + expect(globalState.softRedeployInProgress).to.equal(false); + sinon.restore(); + }); + it('should return early if removal is in progress', async () => { globalState.removalInProgress = true; @@ -583,8 +737,63 @@ describe('advancedWorkflows tests', () => { }); }); - // Note: masterSlaveApps is a recursive function that continuously runs in production. - // These tests use a counter to prevent infinite recursion after the first iteration. + describe('startMasterSlaveApps tests', () => { + const intervalMs = config.fluxapps.masterSlaveIntervalMs; + // A pass on a node that is installing returns at the top, so each one is a + // single count of masterSlave:cycles and nothing else. + const installingNode = () => ({ installationInProgress: true }); + const noApps = () => sinon.stub().resolves({ status: 'success', data: [] }); + let clock; + let cycles; + let loop; + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + cycles = sinon.spy(fluxEventBus, 'count').withArgs('masterSlave:cycles'); + }); + + afterEach(() => { + loop?.stop(); + loop = null; + }); + + it('runs a pass at once, and the next one the interval after it ends', async () => { + loop = advancedWorkflows.startMasterSlaveApps(installingNode(), noApps(), noApps(), https); + await clock.tickAsync(0); + expect(cycles.callCount).to.equal(1); + + await clock.tickAsync(intervalMs - 1); + expect(cycles.callCount).to.equal(1); + await clock.tickAsync(1); + expect(cycles.callCount).to.equal(2); + await clock.tickAsync(intervalMs); + expect(cycles.callCount).to.equal(3); + }); + + it('runs no further pass once stopped', async () => { + loop = advancedWorkflows.startMasterSlaveApps(installingNode(), noApps(), noApps(), https); + await clock.tickAsync(0); + loop.stop(); + + await clock.tickAsync(intervalMs * 3); + expect(cycles.callCount).to.equal(1); + }); + + it('keeps running passes after one throws', async () => { + fluxEventBus.count.restore(); + const count = sinon.stub(fluxEventBus, 'count'); + count.withArgs('masterSlave:cycles').onFirstCall().throws(new Error('count failed')); + const logError = sinon.stub(log, 'error'); + + loop = advancedWorkflows.startMasterSlaveApps(installingNode(), noApps(), noApps(), https); + await clock.tickAsync(0); + expect(logError.calledWithMatch('count failed'), 'the first pass never threw, so this proves nothing').to.equal(true); + + await clock.tickAsync(intervalMs); + expect(count.withArgs('masterSlave:cycles').callCount).to.equal(2); + }); + }); + describe('masterSlaveApps tests', () => { let globalState; let serviceHelperStub; @@ -601,7 +810,6 @@ describe('advancedWorkflows tests', () => { let syncthingStatusStub; let syncthingDevicesStub; let axiosGetStub; - let recursionCounter; // What FDM sends when it IS answering and this app simply has no primary // yet: a success body carrying an empty ips array. test-infra/fdm-stub @@ -611,7 +819,6 @@ describe('advancedWorkflows tests', () => { const fdmNoPrimary = () => ({ data: { status: 'success', data: { ips: [] } } }); beforeEach(() => { - recursionCounter = 0; // every app directory has its volume mounted unless a test says otherwise sinon.stub(volumeService, 'isPathMounted').resolves(true); globalState = require('../../ZelBack/src/services/utils/globalState'); @@ -641,15 +848,8 @@ describe('advancedWorkflows tests', () => { verdict: peerIdentityService.IdentityVerdict.UNVERIFIABLE, status: 404, data: null, mayReadUnsigned: true, }); - // Stub delay to prevent recursive calls - after first call, block recursion - serviceHelperDelayStub = sinon.stub(serviceHelper, 'delay').callsFake(async () => { - recursionCounter += 1; - if (recursionCounter > 1) { - // Prevent recursion by returning a promise that never resolves - return new Promise(() => {}); - } - return Promise.resolve(); - }); + // a wait inside a pass ends at once unless a test says otherwise + serviceHelperDelayStub = sinon.stub(serviceHelper, 'delay').resolves(); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); fluxNetworkHelperStub = sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress'); @@ -841,6 +1041,54 @@ describe('advancedWorkflows tests', () => { } }); + // Electing an app sets its folder's type - sending on a primary, receive-only on + // a standby - and the reconciler starts none of its components while it is held. + // Each hold is checked beside the same app unheld, which the election does decide. + describe('an app the node holds back from election', () => { + const appName = 'testapp'; + const runPass = async () => { + const installedApps = sinon.stub().resolves({ + status: 'success', + data: [{ name: appName, version: 3, containerData: 'g:/data' }], + }); + const listRunningApps = sinon.stub().resolves({ status: 'success', data: [] }); + serviceHelperStub.resolves(fdmNoPrimary()); + await advancedWorkflows.masterSlaveApps(globalState, installedApps, listRunningApps, https); + }; + + it('decides the app when nothing holds it', async () => { + const count = sinon.stub(fluxEventBus, 'count'); + await runPass(); + sinon.assert.calledWith(count, 'masterSlave:decision', sinon.match.string, 'evaluated'); + }); + + it('leaves an app whose removal is pending undecided', async () => { + const count = sinon.stub(fluxEventBus, 'count'); + globalState.departingApps.enter(appName); + try { + await runPass(); + } finally { + globalState.departingApps.leave(appName); + } + sinon.assert.calledWith(count, 'masterSlave:decision', appName, 'skippedDeparting'); + sinon.assert.neverCalledWith(count, 'masterSlave:decision', sinon.match.string, 'evaluated'); + expect(serviceHelperStub.called, 'FDM was asked about an app the election leaves alone').to.be.false; + }); + + it('leaves an app boot has not yet confirmed it holds undecided', async () => { + const count = sinon.stub(fluxEventBus, 'count'); + globalState.unconfirmedApps.set(appName, Symbol(appName)); + try { + await runPass(); + } finally { + globalState.unconfirmedApps.delete(appName); + } + sinon.assert.calledWith(count, 'masterSlave:decision', appName, 'skippedUnconfirmed'); + sinon.assert.neverCalledWith(count, 'masterSlave:decision', sinon.match.string, 'evaluated'); + expect(serviceHelperStub.called, 'FDM was asked about an app the election leaves alone').to.be.false; + }); + }); + // Shared fixture for the recovery tests below: a v3 g: app with this node in // the location list. `peers` are the other nodes, in election order. // `selfRunningSince` places this node in the election order; `receiveOnlyCache` @@ -889,25 +1137,7 @@ describe('advancedWorkflows tests', () => { data: [{ path: `/root/.flux/ZelApps/${appId}`, type: 'sendreceive' }], }); - // masterSlaveApps re-invokes itself from its own finally, so a single call - // otherwise executes the election twice and every count assertion doubles. - // Let exactly one full pass run: the first delay resolves but arms the - // installation gate so the recursive pass returns at the top, and the second - // delay never resolves so the chain stops there. - let delayCalls = 0; - serviceHelperDelayStub.resetBehavior(); - serviceHelperDelayStub.callsFake(async () => { - delayCalls += 1; - if (delayCalls === 1) { - globalState.installationInProgress = true; - return undefined; - } - return new Promise(() => {}); - }); - return async () => { - delayCalls = 0; - globalState.installationInProgress = false; await advancedWorkflows.masterSlaveApps( globalState, installedApps, listRunningApps, https, ); @@ -1062,6 +1292,18 @@ describe('advancedWorkflows tests', () => { expect(linesMatching(logInfo, 'operator-stopped')).to.have.lengthOf(1); }); + it('excludes a component whose stop lock cannot be read, and elects nothing for it', async () => { + const appName = 'unreadlockapp'; + sinon.stub(appsRuntimeState, 'operatorStopState').rejects(new Error('connection timed out')); + const logWarn = sinon.stub(log, 'warn'); + const runPass = electionFixture(appName); + + await runPass(); + + expect(serviceHelperStub.called, 'the election went on to FDM with an unread lock').to.be.false; + expect(linesMatching(logWarn, 'stop lock could not be read')).to.have.lengthOf(1); + }); + it('announces again after the operator lock is lifted and re-applied', async () => { const appName = 'relockapp'; const operatorStopped = sinon.stub(appsRuntimeState, 'operatorStopState'); @@ -1731,6 +1973,321 @@ describe('advancedWorkflows tests', () => { }); }); + // An app with more than one g: component is elected as a whole: its primary + // runs every one of them on one node, and every other holder runs none. + describe('an app with more than one g: component', () => { + const APP = 'twinapp'; + const DB = `db_${APP}`; + const WEB = `web_${APP}`; + const PEER = '192.168.1.90:16127'; + const namesThisNode = { data: { status: 'success', data: { ips: ['192.168.1.5'] } } }; + const namesPeer = { data: { status: 'success', data: { ips: ['192.168.1.90'] } } }; + let promote; + let standDown; + let holdAsStandby; + let holdAsPrimary; + let count; + let logInfo; + + // db and web are elected; cache runs on every instance and is never the + // election's. `running` and `ready` name components by their identifier. + const twinFixture = ({ running = [], ready = [DB, WEB], peers = [PEER] } = {}) => { + dockerServiceStub.callsFake((name) => `flux${name}`); + const installedApps = sinon.stub().resolves({ + status: 'success', + data: [{ + name: APP, + version: 8, + compose: [ + { name: 'db', containerData: 'g:/data' }, + { name: 'cache', containerData: '/cache' }, + { name: 'web', containerData: 'g:/srv' }, + ], + }], + }); + const listRunningApps = sinon.stub().resolves({ + status: 'success', + data: running.map((identifier) => ({ Names: [`/flux${identifier}`] })), + }); + ready.forEach((identifier) => globalState.receiveOnlySyncthingAppsCache.set(`flux${identifier}`, { restarted: true })); + fluxNetworkHelperStub.resolves('192.168.1.5:16127'); + registryManagerStub.resolves([ + { name: APP, ip: '192.168.1.5:16127', runningSince: '2026-01-01T00:00:00.000Z' }, + ...peers.map((ip, n) => ({ name: APP, ip, runningSince: `2026-01-01T00:0${n + 1}:00.000Z` })), + ]); + // no folder sends here, so a copy is ready only when its mark says so + syncthingServiceStub.resolves([]); + let delayCalls = 0; + serviceHelperDelayStub.resetBehavior(); + serviceHelperDelayStub.callsFake(async () => { + delayCalls += 1; + if (delayCalls === 1) { + globalState.installationInProgress = true; + return undefined; + } + return new Promise(() => {}); + }); + return async () => { + delayCalls = 0; + globalState.installationInProgress = false; + await advancedWorkflows.masterSlaveApps(globalState, installedApps, listRunningApps, https); + }; + }; + const peersHold = (held) => { + axiosGetStub.resetBehavior(); + axiosGetStub.callsFake(peerAnswers({ held })); + }; + const promoted = () => promote.args.map(([identifier]) => identifier); + + beforeEach(() => { + promote = sinon.stub(primaryRole, 'promote').returns(true); + standDown = sinon.stub(primaryRole, 'standDown').returns(true); + holdAsStandby = sinon.stub(primaryRole, 'holdAsStandby').resolves(); + holdAsPrimary = sinon.stub(primaryRole, 'holdAsPrimary').resolves(); + count = sinon.stub(fluxEventBus, 'count'); + logInfo = sinon.stub(log, 'info'); + }); + + describe('with no primary named', () => { + beforeEach(() => { + serviceHelperStub.resolves(fdmNoPrimary()); + }); + + it('starts every g: component, in compose order, and no other', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture(); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([DB, WEB]); + }); + + it('starts none while one component\'s copy is not ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ ready: [DB] }); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + expect(linesMatching(logInfo, `not ready yet (syncthing not synced for ${WEB})`)).to.have.lengthOf(1); + }); + + // A peer running only the later component is the app's primary: starting + // the anchor here would split the app across two nodes. + it('starts none when a peer holds only a later g: component', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture(); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + + it('starts the component its operator did not stop, and leaves the stopped one down', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').callsFake(async (identifier) => (identifier === WEB ? LOCKED : UNLOCKED)); + const runPass = twinFixture(); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([DB]); + }); + + it('leaves the app out of the election while its operator has stopped every g: component', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(LOCKED); + const runPass = twinFixture(); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(count, 'masterSlave:decision', DB, 'operatorStopped'); + }); + + describe('on a node that already runs one of them', () => { + it('starts the other once its copy is ready and no peer holds the app', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([WEB]); + }); + + it('does not start the other beside a peer that holds the app', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + + // Canary for the test above: the same pass with the peer unreadable is + // not a clearance either, while FDM names nobody. + it('does not start the other beside a peer it cannot rule out', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + }); + }); + + describe('FDM naming this node', () => { + beforeEach(() => { + serviceHelperStub.resolves(namesThisNode); + }); + + it('keeps the running component sending and starts the one the spec added once its copy is ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB] }); + peersHold([]); + + await runPass(); + + sinon.assert.calledOnceWithExactly(holdAsPrimary, DB, `flux${DB}`); + expect(promoted()).to.deep.equal([WEB]); + }); + + it('waits for the missing component\'s copy to be ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB], ready: [DB] }); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(count, 'masterSlave:decision', DB, 'notReady'); + }); + + it('takes the app over only once every copy is ready', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ ready: [WEB] }); + peersHold([]); + + await runPass(); + + sinon.assert.notCalled(promote); + }); + + it('does not take the app over while another holder holds any of it', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture(); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(count, 'masterSlave:decision', DB, 'namedButHeldElsewhere'); + }); + }); + + describe('FDM naming another node', () => { + beforeEach(() => { + serviceHelperStub.resolves(namesPeer); + }); + + it('stands down every g: component it runs', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + const runPass = twinFixture({ running: [DB, WEB] }); + + await runPass(); + + expect(standDown.args.map(([identifier]) => identifier)).to.deep.equal([DB, WEB]); + sinon.assert.notCalled(promote); + }); + + // A component running here beside one that is not still makes this node + // a holder of the app, and stands down with it. + it('stands down a component that runs here even when the other does not', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + standDown.callsFake((identifier, appId, { running }) => running); + const runPass = twinFixture({ running: [WEB] }); + + await runPass(); + + sinon.assert.calledWith(standDown, WEB, `flux${WEB}`, { running: true }); + sinon.assert.calledOnceWithExactly(holdAsStandby, DB, `flux${DB}`, { othersHold: sinon.match.func }); + }); + + it('leaves a component its operator stopped as its lock keeps it', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').callsFake(async (identifier) => (identifier === WEB ? LOCKED : UNLOCKED)); + const runPass = twinFixture({ running: [DB] }); + + await runPass(); + + expect(standDown.args.map(([identifier]) => identifier)).to.deep.equal([DB]); + sinon.assert.notCalled(holdAsStandby); + }); + + // The question a standby's paused folder asks, and the one the named node + // is asked whether it has decided: both about the whole app. + it('asks the other holders about every g: component of the app', async () => { + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(UNLOCKED); + standDown.returns(false); + const runPass = twinFixture(); + peersHold([`flux${WEB}`]); + + await runPass(); + const { othersHold } = holdAsStandby.firstCall.args[2]; + + expect(await othersHold()).to.equal(PeerComponent.RUNNING); + }); + }); + + describe('started by its operator after a stop on this node as its primary', () => { + beforeEach(() => { + serviceHelperStub.resolves(fdmNoPrimary()); + sinon.stub(appsRuntimeState, 'operatorStopState').resolves(START_REQUESTED_AS_PRIMARY); + sinon.stub(appsRuntimeState, 'releaseOperatorStart').resolves(true); + sinon.stub(appReconciler, 'applyIntent').callsFake(async (id, mutate) => { + await mutate(); + return true; + }); + promote.callsFake((identifier, appId) => { + primaryRoleChanges.set(identifier, { state: 'promoting', appId }); + return true; + }); + }); + + afterEach(() => { + primaryRoleChanges.promotingIdentifiers().forEach((identifier) => primaryRoleChanges.end(identifier, primaryRoleChanges.get(identifier))); + }); + + it('resumes every g: component and lifts each one\'s lock', async () => { + const runPass = twinFixture({ peers: [PEER] }); + peersHold([]); + + await runPass(); + + expect(promoted()).to.deep.equal([DB, WEB]); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, DB); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, WEB); + }); + + // A peer that took the app over decides it for every component: each lock + // is lifted, and none of them starts here. + it('lifts each one\'s lock without starting it when a peer holds the app', async () => { + const runPass = twinFixture({ peers: [PEER] }); + peersHold([`flux${WEB}`]); + + await runPass(); + + sinon.assert.notCalled(promote); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, DB); + sinon.assert.calledWith(appsRuntimeState.releaseOperatorStart, WEB); + }); + }); + }); + describe('isElectedPrimaryHere answers in three states', () => { // Finding 25 of the review: the true path had no coverage at all, and the // false path could not be told apart from "the election never ran". Each @@ -2458,7 +3015,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); }); @@ -2467,7 +3024,6 @@ describe('advancedWorkflows tests', () => { // back within it keeps what it holds. it('will not start beside a silent peer that announced a shutdown and has not passed its expiry', async () => { const appName = 'rebootingpeerapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2491,13 +3047,12 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); }); it('will not start beside a silent peer when this node cannot read which nodes announced a shutdown', async () => { const appName = 'shutdownsunreadableapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2530,7 +3085,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(1); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(1); }); @@ -2555,7 +3110,6 @@ describe('advancedWorkflows tests', () => { // resumes them. The closed connection is this node's doing, not the peer's. it('will not start beside a silent peer whose device this node\'s syncthing has paused', async () => { const appName = 'pausedpeerapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2572,7 +3126,6 @@ describe('advancedWorkflows tests', () => { // that closed before it started says nothing about the peer now. it('will not start beside a silent peer whose connection closed before this node\'s syncthing started', async () => { const appName = 'restartedsyncthingapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2589,7 +3142,6 @@ describe('advancedWorkflows tests', () => { // as the previous syncthing stopped reads the same second as this one's start. it('will not start beside a silent peer whose connection closed in the second this node\'s syncthing started', async () => { const appName = 'samesecondapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2700,7 +3252,6 @@ describe('advancedWorkflows tests', () => { it('will not start beside a silent peer when this node cannot read when its syncthing started', async () => { const appName = 'statusunreadableapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2715,7 +3266,6 @@ describe('advancedWorkflows tests', () => { it('will not start beside a silent peer when this node cannot read whether its syncthing paused it', async () => { const appName = 'devicesunreadableapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); const logInfo = sinon.stub(log, 'info'); const runPass = electionFixture(appName, ['192.168.1.90:16127']); serviceHelperStub.resolves(fdmNoPrimary()); @@ -2778,7 +3328,7 @@ describe('advancedWorkflows tests', () => { await runPass(); expect(linesMatching(logInfo, 'alive, and cannot be ruled out')).to.have.lengthOf(1); - expect(linesMatching(logInfo, 'the component is free there')).to.have.lengthOf(0); + expect(linesMatching(logInfo, 'the app is free there')).to.have.lengthOf(0); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); @@ -3051,7 +3601,6 @@ describe('advancedWorkflows tests', () => { // the node it names may not be the one running it. it('lets a standby\'s paused folder ask every other holder, not only the node FDM names', async () => { const appName = 'fdmholderapp'; - sinon.stub(appsRuntimeState, 'isOperatorStopped').resolves(false); sinon.stub(log, 'info'); const hold = sinon.stub(primaryRole, 'holdAsStandby').resolves(); const runPass = electionFixture(appName, ['192.168.1.90:16137', '192.168.1.91:16127']); @@ -3321,7 +3870,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, `component:${appName} is held on peer node (index 2)`)).to.have.lengthOf(1); + expect(linesMatching(logInfo, `app:${appName} is held on peer node (index 2)`)).to.have.lengthOf(1); expect(linesMatching(logInfo, 'starting docker component')).to.have.lengthOf(0); }); @@ -3573,19 +4122,7 @@ describe('advancedWorkflows tests', () => { ], }); - let delayCalls = 0; - serviceHelperDelayStub.resetBehavior(); - serviceHelperDelayStub.callsFake(async () => { - delayCalls += 1; - if (delayCalls === 1) { - globalState.installationInProgress = true; - return undefined; - } - return new Promise(() => {}); - }); const runPass = async () => { - delayCalls = 0; - globalState.installationInProgress = false; await advancedWorkflows.masterSlaveApps( globalState, installedApps, listRunningApps, https, ); @@ -3607,7 +4144,7 @@ describe('advancedWorkflows tests', () => { await runPass(); - expect(linesMatching(logInfo, `component:${first} is held on peer node (previous primary)`)).to.have.lengthOf(1); + expect(linesMatching(logInfo, `app:${first} is held on peer node (previous primary)`)).to.have.lengthOf(1); // the assertion that discriminates: the pass carried on to the second app expect(linesMatching(logInfo, `starting docker component:${second}`)).to.have.lengthOf(1); }); @@ -4790,10 +5327,8 @@ describe('advancedWorkflows tests', () => { // What this node publishes about the volume being replaced. Peers rank a seed // on it, so it describes one incarnation and must not outlive it. const armHolding = () => { - volGlobalState.folderHoldings = new Map([ - [identifier, { bytes: 5821604997, newestModified: 200 }], - ['fluxother_TestApp', { bytes: 4096, newestModified: 100 }], - ]); + volGlobalState.folderStates.set(identifier, { state: 'measured', bytes: 5821604997, newestModified: 200 }); + volGlobalState.folderStates.set('fluxother_TestApp', { state: 'measured', bytes: 4096, newestModified: 100 }); volGlobalState.seedMarks.set(identifier, { stage: 'intent', bytes: 5821604997, newestModified: 200 }); volGlobalState.seedMarks.set('fluxother_TestApp', { stage: 'intent', bytes: 4096, newestModified: 100 }); }; @@ -4801,7 +5336,7 @@ describe('advancedWorkflows tests', () => { beforeEach(() => { volGlobalState = require('../../ZelBack/src/services/utils/globalState'); volGlobalState.receiveOnlySyncthingAppsCache.clear(); - volGlobalState.folderHoldings = null; + volGlobalState.folderStates.clear(); volGlobalState.seedMarks.clear(); const hwRequirements = require('../../ZelBack/src/services/appRequirements/hwRequirements'); sinon.stub(hwRequirements, 'getNodeSpecs').resolves({ ssdStorage: 10000 }); @@ -4979,12 +5514,12 @@ describe('advancedWorkflows tests', () => { expect(thrown, 'the pre-flight abort did not fire').to.not.equal(null); expect( - volGlobalState.folderHoldings.has(identifier), + volGlobalState.folderStates.get(identifier), 'an aborted pre-flight withdrew a claim whose data is intact', - ).to.equal(true); + ).to.include({ state: 'measured' }); }); - it('drops the published holding at the point of no return', async () => { + it('records the volume as not looked at from the point of no return', async () => { // Once the allocation runs the volume is empty. A claim surviving from the // previous incarnation describes data this node no longer has, and peers rank // the seed on it - so it outranks a node that still holds the app and seeds @@ -5004,11 +5539,11 @@ describe('advancedWorkflows tests', () => { expect(thrown, 'the flow never reached the allocation').to.not.equal(null); expect(thrown.message, 'the flow aborted before the allocation').to.equal('fallocate blocked by test'); expect( - volGlobalState.folderHoldings.has(identifier), + volGlobalState.folderStates.get(identifier), 'the point of no return left a claim for a volume that is now empty', - ).to.equal(false); + ).to.deep.equal({ state: 'notLookedAt' }); // Scoped to the volume being replaced, not a clear of every answer the node holds. - expect(volGlobalState.folderHoldings.has('fluxother_TestApp')).to.equal(true); + expect(volGlobalState.folderStates.get('fluxother_TestApp')).to.include({ state: 'measured' }); expect(volGlobalState.seedMarks.has(identifier), 'the point of no return left an offer to seed an empty volume').to.equal(false); expect(volGlobalState.seedMarks.has('fluxother_TestApp')).to.equal(true); }); @@ -5161,6 +5696,42 @@ describe('advancedWorkflows tests', () => { expect(globalState.backupInProgress).to.not.include(appname); }); + // A backup ends by starting the app, so it is refused for an app the node is + // removing or has not yet confirmed it still holds. + it('refuses an app whose removal is pending', async () => { + const res = makeRes(); + globalState.departingApps.enter(appname); + let result; + try { + result = await advancedWorkflows.appendBackupTask(backupReq(), res); + } finally { + globalState.departingApps.leave(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is being removed from this node'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.backupInProgress).to.not.include(appname); + }); + + it('refuses an app boot has not yet confirmed it holds', async () => { + const res = makeRes(); + globalState.unconfirmedApps.set(appname, Symbol(appname)); + let result; + try { + result = await advancedWorkflows.appendBackupTask(backupReq(), res); + } finally { + globalState.unconfirmedApps.delete(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is held until this node confirms it still holds it'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.backupInProgress).to.not.include(appname); + }); + it('refuses cleanly when the app has no specification, not with a TypeError', async () => { // The restore guards this; the backup read the spec and handed it straight // to syncedComponentsOfApp, which threw on null deep in the flow instead of @@ -5545,6 +6116,42 @@ describe('advancedWorkflows tests', () => { expect(globalState.restoreInProgress).to.not.include(appname); }); + // A restore ends by starting the app, so it is refused for an app the node + // is removing or has not yet confirmed it still holds. + it('refuses an app whose removal is pending', async () => { + const res = makeRes(); + globalState.departingApps.enter(appname); + let result; + try { + result = await advancedWorkflows.appendRestoreTask(restoreReq(), res); + } finally { + globalState.departingApps.leave(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is being removed from this node'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.restoreInProgress).to.not.include(appname); + }); + + it('refuses an app boot has not yet confirmed it holds', async () => { + const res = makeRes(); + globalState.unconfirmedApps.set(appname, Symbol(appname)); + let result; + try { + result = await advancedWorkflows.appendRestoreTask(restoreReq(), res); + } finally { + globalState.unconfirmedApps.delete(appname); + } + + expect(result).to.equal(false); + expect(res.write.getCalls().map((c) => c.args[0]).join(' ')).to.include('is held until this node confirms it still holds it'); + sinon.assert.notCalled(dockerService.appDockerStop); + sinon.assert.notCalled(dockerService.appDockerStart); + expect(globalState.restoreInProgress).to.not.include(appname); + }); + it('refuses a component the app does not have', async () => { const req = restoreReq(); req.body.restore = [{ component: 'palworld; rm -rf /', restore: true, url: 'https://example.invalid/a.tar.gz' }]; @@ -6297,6 +6904,43 @@ describe('advancedWorkflows tests', () => { expect(globalState.softRedeployInProgress).to.be.false; }); + it('leaves the app down with its data when the step after the removal cannot read the database', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); + sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + + await advancedWorkflows.softRedeploy(v3Spec, null); + + expect(removeAppLocally.called, 'the app was uninstalled because the database did not answer').to.be.false; + expect(globalState.softRedeployInProgress).to.be.false; + }); + + // A soft redeploy keeps the app's volume, and with it the node's record of + // when it took the app, whichever way it ends. + it('leaves the placement record alone when the app was removed and the reinstall did not happen', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves(v3Spec); + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves(); + sinon.stub(appUninstaller, 'softUninstallApplication').resolves(); + // the node is taken by another install by the time the reinstall asks for it + sinon.stub(appInstaller, 'checkAppRequirements').callsFake(async () => { + globalState.installationInProgress = true; + return true; + }); + const removePlacement = sinon.stub(appPlacements, 'removePlacement').resolves(); + const recordPlacement = sinon.stub(appPlacements, 'recordPlacement').resolves(); + + try { + await advancedWorkflows.softRedeploy(v3Spec, null); + } finally { + globalState.installationInProgress = false; + } + + sinon.assert.notCalled(removePlacement); + sinon.assert.notCalled(recordPlacement); + }); + it('records that a failure after the app is down still force-uninstalls it', async () => { // Pins CURRENT behaviour, not desired behaviour. The gate covers a removal // that never finished; past that point a failure still force-uninstalls @@ -6725,6 +7369,28 @@ describe('advancedWorkflows tests', () => { expect(res.writesAfterEnd).to.be.empty; }); + it('does not uninstall the app when the step after a soft component removal cannot read the database', async () => { + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + const res = streamingRes(); + + await advancedWorkflows.redeployComponentAPI(req, res); + + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.writesAfterEnd).to.be.empty; + }); + + it('does not uninstall the app when the step after a hard component removal cannot read the database', async () => { + sinon.stub(appInstaller, 'checkAppRequirements').rejects(new MongoNetworkTimeoutError('connection timed out')); + sinon.stub(appUninstaller, 'hardUninstallComponent').resolves(); + req.params.force = 'true'; + const res = streamingRes(); + + await advancedWorkflows.redeployComponentAPI(req, res); + + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.writesAfterEnd).to.be.empty; + }); + // The gate must not be shut. A teardown that COMPLETED and a failure after it // is the case the forced uninstall exists for, and a suite that proved only // the refusals would pass with the fallback deleted altogether. @@ -6870,6 +7536,18 @@ describe('advancedWorkflows tests', () => { // redeploys nothing. Nothing on that branch judges what it wrote: the installer // judges what it installs, and this installs nothing, so an owner the network // refuses holds the application until something unrelated sweeps the node. + it('leaves an app whose removal is pending to a later pass', async () => { + const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + globalState.departingApps.enter(installedApp.name); + try { + await advancedWorkflows.reinstallOldApplications(); + } finally { + globalState.departingApps.leave(installedApp.name); + } + + expect(softUninstallComponent.called, 'rebuilt an app its pending removal would take again').to.be.false; + }); + it('asks the sweep about an application whose record it rewrote without redeploying', async () => { const requestComplianceSweep = sinon.stub(imageManager, 'requestComplianceSweep').resolves(); dbHelper.findOneInDatabase.resolves({ @@ -6900,6 +7578,131 @@ describe('advancedWorkflows tests', () => { ).to.be.false; }); + // A version 8 app whose components change is removed and installed again as + // a whole, and the network is told when it does not come back. + describe('a redeploy for a change of components', () => { + const v8Installed = { ...installedApp, version: 8, enterprise: '' }; + const v8New = { + ...v8Installed, + hash: 'newhash', + compose: [...v8Installed.compose, { ...v8Installed.compose[0], name: 'db', ports: ['31001'] }], + }; + let removeAppLocally; + let registerAppLocally; + let checkAppRequirements; + let announceRemoval; + + beforeEach(() => { + dbHelper.findInDatabase.resolves([v8Installed]); + dbHelper.findOneInDatabase.resolves(v8New); + checkAppRequirements = sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(RemovalOutcome.REMOVED); + announceRemoval = sinon.stub(appUninstaller, 'announceRemoval').resolves(); + registerAppLocally = sinon.stub(appInstaller, 'registerAppLocally').resolves(InstallOutcome.INSTALLED); + }); + + it('removes the app and installs it again', async () => { + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.callOrder(removeAppLocally, checkAppRequirements, registerAppLocally); + sinon.assert.notCalled(announceRemoval); + }); + + it('tells the network when the reinstall does not bring the app back', async () => { + registerAppLocally.resolves(InstallOutcome.FAILED); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnceWithExactly(announceRemoval, 'myapp'); + }); + + it('tells the network when the new specification\'s requirements are not met after the removal', async () => { + checkAppRequirements.rejects(new Error('Insufficient RAM')); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(removeAppLocally); + sinon.assert.notCalled(registerAppLocally); + sinon.assert.calledOnceWithExactly(announceRemoval, 'myapp'); + }); + }); + + it('does not uninstall an app whose redeploy stops on a database that does not answer', async () => { + sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + + await advancedWorkflows.reinstallOldApplications(); + + expect(removeAppLocally.called, 'an app was uninstalled with its data because the database did not answer').to.be.false; + }); + + it('never deletes the app\'s row: the new specification replaces it in one write', async () => { + sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + const deleteRow = sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({}); + const replaceRow = sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + + await advancedWorkflows.reinstallOldApplications(); + + expect(replaceRow.called, 'the fixture never reached the row write').to.be.true; + expect(deleteRow.called, 'the row was deleted, so a failed write leaves the app with none').to.be.false; + }); + + describe('an app upgraded from the old format', () => { + const oldFormat = { + version: 3, name: 'myapp', hash: 'oldhash', repotag: 'nginx:1.0', hdd: 5, + }; + + it('is rebuilt from nothing at its new specification', async () => { + const rebuildApp = sinon.stub(appRebuild, 'rebuildApp').resolves('installed'); + dbHelper.findInDatabase.resolves([oldFormat]); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(rebuildApp); + expect(rebuildApp.firstCall.args[0]).to.include({ name: 'myapp', hash: 'newhash', version: 4 }); + }); + + it('says when the pass has ended', async () => { + sinon.stub(appRebuild, 'rebuildApp').resolves('installed'); + const publish = sinon.stub(fluxEventBus, 'publish'); + dbHelper.findInDatabase.resolves([oldFormat]); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledWith(publish, 'reinstall:passEnded', {}); + }); + + it('lets the pass go on to the next app when its rebuild does not complete', async () => { + const rebuildApp = sinon.stub(appRebuild, 'rebuildApp').resolves('failed'); + const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); + sinon.stub(dbHelper, 'replaceOneInDatabase').resolves({ acknowledged: true }); + const other = { ...installedApp, name: 'otherapp' }; + dbHelper.findInDatabase.resolves([oldFormat, other]); + dbHelper.findOneInDatabase.callsFake(async (db, collection, query) => (query.name === 'otherapp' + ? { ...newSpec, name: 'otherapp' } : newSpec)); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(rebuildApp); + expect(softUninstallComponent.called, 'the pass stopped at the app whose rebuild failed').to.equal(true); + }); + }); + + it('a canary: a redeploy that fails for another reason still uninstalls the app', async () => { + sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); + const removeAppLocally = sinon.stub(appUninstaller, 'removeAppLocally').resolves(); + sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new Error('Insufficient RAM')); + + await advancedWorkflows.reinstallOldApplications(); + + sinon.assert.calledOnce(removeAppLocally); + }); + it('declines a second pass that starts before the first has announced itself', async () => { const softUninstallComponent = sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); @@ -7092,17 +7895,18 @@ describe('advancedWorkflows tests', () => { // The node records the new specification before it builds the replacement, // so the row cannot say the reinstall is done. The announcement does, once - // every replaced component is back and the app restarted. + // every replaced component is back and the app's restart requested. describe('announcing the replaced components', () => { let publish; - let appDockerRestart; + let dockerRestart; + let requestRestart; + let applyIntent; let installApplicationSoft; beforeEach(() => { sinon.stub(appUninstaller, 'softUninstallComponent').resolves(); sinon.stub(appUninstaller, 'removeAppLocally').resolves(); - sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({}); - sinon.stub(dbHelper, 'insertOneToDatabase').resolves({ acknowledged: true }); + sinon.stub(dbHelper, 'replaceOneInDatabase').resolves({ acknowledged: true }); sinon.stub(appInstaller, 'checkAppRequirements').resolves(true); installApplicationSoft = sinon.stub(appInstaller, 'installApplicationSoft').resolves(); sinon.stub(appNetworkLinker, 'checkAppNetworkRequirements').resolves(); @@ -7110,13 +7914,18 @@ describe('advancedWorkflows tests', () => { sinon.stub(registryManager, 'getApplicationSpecifications').resolves(newSpec); sinon.stub(volumeService, 'ensureMountPathsExist').resolves(); sinon.stub(appInspector, 'startAppMonitoring'); - appDockerRestart = sinon.stub(dockerServiceModule, 'appDockerRestart').resolves(); + dockerRestart = sinon.stub(dockerServiceModule, 'appDockerRestart').resolves(); + requestRestart = sinon.stub(appsRuntimeState, 'requestRestart').resolves(); + applyIntent = sinon.stub(appReconciler, 'applyIntent').callsFake(async (identifier, mutate) => { + await mutate(); + return true; + }); publish = sinon.spy(fluxEventBus, 'publish'); }); const redeployedEvents = () => publish.getCalls().filter((c) => c.args[0] === 'app:componentRedeployed'); - it('announces each replaced component after the app is back and restarted', async () => { + it('announces each replaced component after the app is back and its restart requested', async () => { await advancedWorkflows.reinstallOldApplications(); const events = redeployedEvents(); @@ -7124,8 +7933,32 @@ describe('advancedWorkflows tests', () => { expect(events[0].args[1]).to.deep.equal({ name: 'myapp', component: 'web', identifier: 'web_myapp', hard: false, }); - sinon.assert.calledWith(appDockerRestart, 'web_myapp'); - expect(appDockerRestart.lastCall.calledBefore(events[0]), 'announced after the restart').to.equal(true); + sinon.assert.calledWith(requestRestart, 'web_myapp'); + expect(requestRestart.lastCall.calledBefore(events[0]), 'announced after the restart is requested').to.equal(true); + }); + + // The reconciler decides what runs on this node: it bounces a component it + // keeps running and starts none it keeps stopped, so a standby's g: + // component is not started beside its primary's. + it('restarts the app through the reconciler, never by docker', async () => { + await advancedWorkflows.reinstallOldApplications(); + + expect(requestRestart.called, 'the fixture reached the restart').to.equal(true); + sinon.assert.calledWith(applyIntent, 'web_myapp', sinon.match.func); + expect(applyIntent.firstCall.calledBefore(requestRestart.firstCall), 'the request is written inside the reconciler\'s slot').to.equal(true); + expect(dockerRestart.called, 'docker restarted a container').to.equal(false); + }); + + it('requests a restart of every component of the app, its g: components included', async () => { + registryManager.getApplicationSpecifications.resolves({ + ...newSpec, + compose: [...newSpec.compose, { ...newSpec.compose[0], name: 'db', containerData: 'g:/data' }], + }); + + await advancedWorkflows.reinstallOldApplications(); + + expect(requestRestart.args.map(([identifier]) => identifier)).to.deep.equal(['web_myapp', 'db_myapp']); + expect(applyIntent.args.map(([identifier]) => identifier)).to.deep.equal(['web_myapp', 'db_myapp']); }); it('announces nothing when a component does not come back', async () => { @@ -7322,6 +8155,52 @@ describe('giving up an app: one pass, two reasons, one safety gate', function () expect(decision.reason).to.equal('SURPLUS'); }); + // An app with more than one g: component is written by the node that runs + // any of them: its primary runs them together. + describe('an app with more than one g: component', () => { + const twoWriters = { + name: 'a', instances: 2, version: 8, compose: [{ name: 'w', containerData: 'g:/data' }, { name: 'v', containerData: 'g:/srv' }], + }; + const runsOnly = (identifier) => sinon.stub().callsFake(async (each) => each === identifier); + + it('leaves the newest copy alone when it runs only a later g: component', async () => { + const decision = await advancedWorkflows.reasonToGiveUpApp( + twoWriters, + locations('5.6.7.8:16127', '9.9.9.9:16127', LOCAL), + LOCAL, + { isComponentRunningLocally: runsOnly('v_a'), liveness: {} }, + ); + + expect(decision.giveUp).to.equal(false); + expect(decision.reason).to.equal('SURPLUS'); + }); + + // Canary for the test above: the same newest copy running none of them trims. + it('trims the newest copy when it runs none of them', async () => { + const decision = await advancedWorkflows.reasonToGiveUpApp( + twoWriters, + locations('5.6.7.8:16127', '9.9.9.9:16127', LOCAL), + LOCAL, + { isComponentRunningLocally: runsOnly('other_a'), liveness: {} }, + ); + + expect(decision.giveUp).to.equal(true); + }); + + it('trims the second-newest once the newest confirms it holds a later g: component', async () => { + newestSays({ held: [dockerServiceModule.getAppIdentifier('v_a')] }); + + const decision = await advancedWorkflows.reasonToGiveUpApp( + twoWriters, + locations('5.6.7.8:16127', LOCAL, '9.9.9.9:16127'), + LOCAL, + { isComponentRunningLocally: sinon.stub().resolves(false), liveness: {} }, + ); + + expect(decision.giveUp).to.equal(true); + }); + }); + it('trims the second-newest once the newest confirms it holds the writer', async () => { newestSays({ held: [writerAppId] }); @@ -7988,6 +8867,17 @@ describe('a redeploy asks what it may rebuild before it takes anything down', () expect(res.write.firstCall.args[0]).to.match(/policy/i); }); + // A rebuilt app starts, and boot has yet to decide whether this one may. + it('declines while boot has not confirmed the node still holds the app, and leaves it where it is', async () => { + globalState.unconfirmedApps.set('plainapp', Symbol('plainapp')); + + const proceed = await advancedWorkflows.mayTearDownToRebuild({ name: 'plainapp', nodes: [] }, res); + + expect(proceed, 'the teardown must not start').to.be.false; + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.write.firstCall.args[0]).to.include('redeploy deferred'); + }); + it('hands back an app whose spec names other nodes, and tells the network', async () => { const proceed = await advancedWorkflows.mayTearDownToRebuild(pinnedSpec(['10.0.0.9:16127']), res); @@ -8091,6 +8981,16 @@ describe('a redeploy asks what it may rebuild before it takes anything down', () expect(globalState.hardRedeployInProgress).to.be.false; }); + it('hardRedeploy leaves a running app alone when it fails before the teardown', async () => { + sinon.stub(globalState, 'operationHolding').throws(new Error('connection timed out')); + + await advancedWorkflows.hardRedeploy({ version: 4, name: 'plainapp', nodes: [] }, res); + + sinon.assert.notCalled(appUninstaller.removeAppLocally); + expect(res.write.lastCall.args[0]).to.match(/before the app was taken down/); + expect(globalState.hardRedeployInProgress).to.be.false; + }); + it('a canary: hardRedeploy with the gate open reaches the teardown', async () => { await advancedWorkflows.hardRedeploy({ version: 4, name: 'plainapp', nodes: [] }, res); diff --git a/tests/unit/apiServer.test.js b/tests/unit/apiServer.test.js index a55be53f82..d86b82512e 100644 --- a/tests/unit/apiServer.test.js +++ b/tests/unit/apiServer.test.js @@ -12,6 +12,7 @@ const globalState = require('../../ZelBack/src/services/utils/globalState'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const fluxCommunicationMessagesSender = require('../../ZelBack/src/services/fluxCommunicationMessagesSender'); const messageStore = require('../../ZelBack/src/services/appMessaging/messageStore'); +const { AppSyncOrchestrator } = require('../../ZelBack/src/services/appMessaging/appSyncOrchestrator'); const { PM2_KILL_TIMEOUT_MS } = require('../../ZelBack/src/services/fluxService'); const serviceManager = require('../../ZelBack/src/services/serviceManager'); const systemdNotify = require('../../ZelBack/src/services/utils/systemdNotify'); @@ -613,6 +614,8 @@ describe('handleSigterm drains syncthing folders before it exits', () => { drainStub = sinon.stub(syncthingService, 'drainFoldersToPeers').resolves([]); sinon.stub(syncthingFolderWrites, 'pauseAllFolders').resolves({ paused: [], failed: [] }); sinon.stub(verifyPool, 'stop'); + // A database another test file left connected would take the write. + sinon.stub(AppSyncOrchestrator, 'writeShutdownReason').resolves(); exitStub = sinon.stub(process, 'exit'); // The flag is one-way and module-wide; set for real it would refuse every // container start in the rest of this mocha process. diff --git a/tests/unit/appController.test.js b/tests/unit/appController.test.js index fe69e35582..e172305265 100644 --- a/tests/unit/appController.test.js +++ b/tests/unit/appController.test.js @@ -361,6 +361,71 @@ describe('appController tests', () => { }); }); + // An app with more than one g: component is elected as a whole, so its + // operator commands defer to the election for each of them, and a lock taken + // on the node running any of them is the primary's. + describe('an app with more than one g: component', () => { + const twinApp = { + name: 'Twin', + version: 4, + compose: [{ name: 'db', containerData: 'g:/data' }, { name: 'cache', containerData: '/c' }, { name: 'web', containerData: 'g:/srv' }], + }; + let requestOperatorStart; + + beforeEach(() => { + verificationHelperStub.resolves(true); + requestOperatorStart = sinon.stub(appsRuntimeState, 'requestOperatorStart').resolves(true); + sinon.stub(fluxEventBus, 'publish'); + }); + + it('leaves every g: component\'s lock for the election to lift', async () => { + stubInstalledApp(twinApp); + + await appController.appStart({ params: { appname: 'Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + expect(requestOperatorStart.args.map(([id]) => id)).to.deep.equal(['db_Twin', 'web_Twin']); + expect(appsRuntimeState.setOperatorStopped.args.map(([id]) => id)).to.deep.equal(['cache_Twin']); + }); + + it('takes each g: component\'s lock as the primary\'s when only one of them runs here', async () => { + stubInstalledApp(twinApp); + sinon.stub(appReconciler, 'clearControllerDesired'); + sinon.stub(appReconciler, 'committedIdentifiers').returns([]); + appReconciler.dockerActual.callsFake(async (id) => ({ reachable: true, exists: true, running: id === 'web_Twin' })); + + await appController.appStop({ params: { appname: 'Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + const options = Object.fromEntries(appsRuntimeState.setOperatorStopped.args.map(([id, , opts]) => [id, opts])); + expect(options).to.deep.equal({ + db_Twin: { force: false, asPrimary: true }, cache_Twin: { force: false }, web_Twin: { force: false, asPrimary: true }, + }); + }); + + it('takes a single g: component\'s lock as the primary\'s on the node running the other', async () => { + stubInstalledApp(twinApp); + sinon.stub(appReconciler, 'clearControllerDesired'); + sinon.stub(appReconciler, 'committedIdentifiers').returns([]); + appReconciler.dockerActual.callsFake(async (id) => ({ reachable: true, exists: true, running: id === 'db_Twin' })); + + await appController.appStop({ params: { appname: 'web_Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + sinon.assert.calledOnceWithExactly(appsRuntimeState.setOperatorStopped, 'web_Twin', true, { force: false, asPrimary: true }); + }); + + // Canary for the test above: on a node holding none of the app the same + // stop is not the primary's. + it('takes it as not the primary\'s on a node running none of them', async () => { + stubInstalledApp(twinApp); + sinon.stub(appReconciler, 'clearControllerDesired'); + sinon.stub(appReconciler, 'committedIdentifiers').returns([]); + appReconciler.dockerActual.resolves({ reachable: true, exists: true, running: false }); + + await appController.appStop({ params: { appname: 'web_Twin' }, query: {} }, { json: sinon.fake((param) => param) }); + + sinon.assert.calledOnceWithExactly(appsRuntimeState.setOperatorStopped, 'web_Twin', true, { force: false, asPrimary: false }); + }); + }); + it('should report an unreachable docker rather than claiming a start', async () => { verificationHelperStub.resolves(true); stubInstalledApp({ diff --git a/tests/unit/appHashSyncService.test.js b/tests/unit/appHashSyncService.test.js index 386b68ce92..6a6f3a79f3 100644 --- a/tests/unit/appHashSyncService.test.js +++ b/tests/unit/appHashSyncService.test.js @@ -5,6 +5,9 @@ const proxyquire = require('proxyquire').noCallThru(); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { Readable } = require('stream'); const config = require('config'); +const { appSyncEvents: syncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); + +const MESSAGES = config.database.appsglobal.collections.appsMessages; function makeStreamResponse(data) { const json = JSON.stringify({ status: 'success', data }); @@ -160,7 +163,19 @@ describe('appHashSyncService tests', () => { describe('triggerAppHashesCheckAPI', () => { - it('should trigger sync when authorized', async () => { + // queued for the sync orchestrator, which runs it once no hash sync or registry + // build holds the node + let requested; + const onRequested = () => { requested += 1; }; + beforeEach(() => { + requested = 0; + syncEvents.on(SYNC_EVENTS.HASH_CHECK_REQUESTED, onRequested); + }); + afterEach(() => { + syncEvents.removeListener(SYNC_EVENTS.HASH_CHECK_REQUESTED, onRequested); + }); + + it('should queue the check when authorized, running nothing itself', async () => { const req = {}; const res = { json: sinon.stub() }; @@ -171,6 +186,8 @@ describe('appHashSyncService tests', () => { expect(res.json.calledOnce).to.be.true; expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req))).to.be.true; + expect(requested, 'the check was not queued').to.equal(1); + sinon.assert.notCalled(dbHelperStub.findInDatabase); }); it('should deny unauthorized access', async () => { @@ -183,6 +200,7 @@ describe('appHashSyncService tests', () => { expect(res.json.calledOnce).to.be.true; expect(messageHelperStub.errUnauthorizedMessage.calledOnce).to.be.true; + expect(requested, 'an unauthorized caller queued a check').to.equal(0); }); it('should handle errors gracefully', async () => { @@ -834,6 +852,7 @@ describe('appHashSyncService tests', () => { const { recentRetry, oldRetry } = makeExpiryTestHashes(); const both = [recentRetry, oldRetry]; dbHelperStub.findInDatabase.resolves(both); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([]); advanceDateOnDelay(); try { @@ -854,6 +873,7 @@ describe('appHashSyncService tests', () => { dbHelperStub.findInDatabase.onFirstCall().resolves([recentRetry, oldRetry, ...padding]); dbHelperStub.findInDatabase.resolves([recentRetry, oldRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([]); serviceHelperStub.axiosGet.callsFake((url) => { if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse([])); @@ -869,6 +889,59 @@ describe('appHashSyncService tests', () => { assertExpiryBehavior(); }); + + // A hash whose message this node holds waits on no peer: the next pass writes its + // registry row from the stored copy, so the ladder that spaces out asking peers + // does not apply to it. + describe('a hash whose message is already stored', () => { + const backedOff = (hash) => collectionStub.bulkWrite.getCalls().some((call) => call.args[0] + .some((op) => op.updateOne?.filter?.hash === hash && op.updateOne.update.$set.syncAttempts !== undefined)); + + it('is not backed off', async () => { + const { recentRetry } = makeExpiryTestHashes(); + dbHelperStub.findInDatabase.resolves([recentRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([{ hash: recentRetry.hash }]); + + advanceDateOnDelay(); + try { + await appHashSyncService.syncMissingHashes({ currentHeight: 2555000 }); + } finally { + Date.now.restore(); + } + + expect(backedOff(recentRetry.hash), 'a hash whose message is stored was backed off').to.equal(false); + }); + + it('a canary: the same hash with no message stored is backed off', async () => { + const { recentRetry } = makeExpiryTestHashes(); + dbHelperStub.findInDatabase.resolves([recentRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([]); + + advanceDateOnDelay(); + try { + await appHashSyncService.syncMissingHashes({ currentHeight: 2555000 }); + } finally { + Date.now.restore(); + } + + expect(backedOff(recentRetry.hash)).to.equal(true); + }); + + it('is not given up on, however long it has waited', async () => { + const { oldRetry } = makeExpiryTestHashes(); + dbHelperStub.findInDatabase.resolves([oldRetry]); + dbHelperStub.findInDatabase.withArgs(sinon.match.any, MESSAGES).resolves([{ hash: oldRetry.hash }]); + + advanceDateOnDelay(); + try { + await appHashSyncService.syncMissingHashes({ currentHeight: 2555000 }); + } finally { + Date.now.restore(); + } + + expect(messageVerifierStub.appHashHasMessageNotFound.called, 'a hash whose message is stored was given up on').to.equal(false); + }); + }); }); describe('processMessages via bulk fetch', () => { @@ -878,8 +951,10 @@ describe('appHashSyncService tests', () => { let localMessageVerifierStub; let localCheckAndDecryptStub; let localLogStub; + let localGlobalState; beforeEach(() => { + localGlobalState = { dbReady: true }; localCollectionStub = { bulkWrite: sinon.stub().resolves({ ok: 1 }), insertMany: sinon.stub().resolves({ insertedCount: 0 }), @@ -902,6 +977,7 @@ describe('appHashSyncService tests', () => { localMessageVerifierStub = { alignStoredMessagesWithPayments: sinon.stub().resolves([]), checkAndRequestApp: sinon.stub().resolves(true), + storeGoverningAppSpecifications: sinon.stub().resolves(), appHashHasMessage: sinon.stub().resolves(), appHashHasMessageNotFound: sinon.stub().resolves(), checkAndRequestMultipleApps: sinon.stub().resolves(), @@ -940,6 +1016,94 @@ describe('appHashSyncService tests', () => { '../fluxCommunicationUtils': { deterministicFluxList: sinon.stub().resolves([]) }, '../fluxCommunication': { openEphemeralConnection: sinon.stub().resolves(null) }, '../fluxNetworkHelper': { getLocalSocketAddress: sinon.stub().resolves('10.0.0.99:16127') }, + '../utils/globalState': localGlobalState, + }); + }); + + describe('the registry rows of the apps it stores', () => { + const streamOne = () => { + const bulkMessages = [{ + type: 'fluxappregister', version: 1, hash: 'hash5', timestamp: Date.now(), + signature: 'sig5', appSpecifications: { name: 'lateapp', version: 4, owner: 'owner' }, + valueSat: 105, txid: 'tx5', height: 1005, + }]; + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1000 + i, value: 100 + i, message: false, + })); + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + return Promise.resolve(getMissingCalls === 1 ? manyMissing : []); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse(bulkMessages)); + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + }; + // the write that marks a hash's message as held + const hashMarks = () => localCollectionStub.bulkWrite.getCalls() + .filter((call) => call.args[0].some((op) => op.updateOne?.update?.$set?.message === true)); + + it('writes them on a node whose registry is built, before it marks their hashes', async () => { + streamOne(); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.called, 'the fixture stored nothing').to.be.true; + sinon.assert.calledOnceWithExactly(localMessageVerifierStub.storeGoverningAppSpecifications, ['lateapp']); + expect(hashMarks(), 'the hash was never marked').to.have.length(1); + expect(localMessageVerifierStub.storeGoverningAppSpecifications.calledBefore(hashMarks()[0])).to.equal(true); + }); + + it('leaves them to the full rebuild on a node whose registry is not built yet', async () => { + localGlobalState.dbReady = false; + streamOne(); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.called, 'the fixture stored nothing').to.be.true; + sinon.assert.notCalled(localMessageVerifierStub.storeGoverningAppSpecifications); + expect(hashMarks()).to.have.length(1); + }); + + it('writes the rows of only the apps whose messages the insert stored', async () => { + const bulkMessages = ['firstapp', 'secondapp'].map((name, i) => ({ + type: 'fluxappregister', version: 1, hash: `hash${5 + i}`, timestamp: Date.now(), + signature: `sig${5 + i}`, appSpecifications: { name, version: 4, owner: 'owner' }, + valueSat: 105 + i, txid: `tx${5 + i}`, height: 1005 + i, + })); + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1000 + i, value: 100 + i, message: false, + })); + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + return Promise.resolve(getMissingCalls === 1 ? manyMissing : []); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse(bulkMessages)); + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + const partial = new Error('duplicate key'); + partial.result = { insertedCount: 1 }; + localCollectionStub.insertMany.rejects(partial); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.firstCall.args[0], 'the fixture inserted fewer than two').to.have.length(2); + sinon.assert.calledOnceWithExactly(localMessageVerifierStub.storeGoverningAppSpecifications, ['firstapp']); + }); + + it('marks no hash whose app\'s row could not be written, so the next retry writes it', async () => { + localMessageVerifierStub.storeGoverningAppSpecifications.rejects(new Error('connection timed out')); + streamOne(); + + await localModule.syncMissingHashes(); + + expect(localCollectionStub.insertMany.called, 'the fixture stored nothing').to.be.true; + expect(hashMarks()).to.deep.equal([]); }); }); @@ -1090,12 +1254,59 @@ describe('appHashSyncService tests', () => { expect(inserted.valueSat).to.equal(105); }); + describe('an enterprise registration', () => { + const syncRegistration = async (change) => { + const bulkMessages = [{ + type: 'fluxappregister', version: 1, hash: 'hash5', timestamp: Date.now(), signature: 'sig5', + appSpecifications: { + name: 'entapp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'blob', ...change, + }, + valueSat: 1e8, txid: 'tx5', height: 1005, + }]; + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1000 + i, value: 100 + i, message: false, + })); + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + if (getMissingCalls === 1) return Promise.resolve(manyMissing); + return Promise.resolve([]); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) return Promise.resolve(makeStreamResponse(bulkMessages)); + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + + await localModule.syncMissingHashes(); + }; + + it('should decrypt a registration with a valid app name and owner', async () => { + await syncRegistration({}); + + sinon.assert.calledOnce(localCheckAndDecryptStub); + expect(localCollectionStub.insertMany.firstCall.args[0][0].hash).to.equal('hash5'); + }); + + [ + ['an invalid app name', { name: 'ent_app' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should store a registration with ${description} without decrypting it`, async () => { + await syncRegistration(change); + + sinon.assert.notCalled(localCheckAndDecryptStub); + expect(localCollectionStub.insertMany.firstCall.args[0][0].hash).to.equal('hash5'); + }); + }); + }); + it('should decrypt prevSpec for enterprise v8 updates before signature verification', async () => { const bulkMessages = [ { type: 'fluxappupdate', version: 8, hash: 'hash1', timestamp: Date.now(), signature: 'sig1', - appSpecifications: { name: 'enterpriseApp', version: 8, owner: 'owner1', enterprise: 'encBlob' }, + appSpecifications: { name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'encBlob' }, valueSat: 1e8, txid: 'tx1', height: 2000, }, ]; @@ -1122,7 +1333,7 @@ describe('appHashSyncService tests', () => { // Previous spec is enterprise v8 with encrypted data const prevSpec = { - name: 'enterpriseApp', version: 8, owner: 'owner1', enterprise: 'prevEncBlob', + name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'prevEncBlob', }; localCollectionStub.find.returns({ project: sinon.stub().returns({ @@ -1138,7 +1349,7 @@ describe('appHashSyncService tests', () => { }); const decryptedPrevSpec = { - name: 'enterpriseApp', version: 8, owner: 'owner1', + name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', compose: [{ name: 'comp1', repotag: 'repo/tag' }], }; @@ -1161,7 +1372,64 @@ describe('appHashSyncService tests', () => { const verifyCall = localMessageVerifierStub.verifyAppMessageUpdateSignature.firstCall; expect(verifyCall).to.not.be.null; // The owner param (6th arg) should be from the decrypted prevSpec - expect(verifyCall.args[5]).to.equal('owner1'); + expect(verifyCall.args[5]).to.equal('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC'); + }); + + it('should verify an update against the stored prevSpec when its owner is invalid, without decrypting it', async () => { + const bulkMessages = [ + { + type: 'fluxappupdate', version: 8, hash: 'hash1', timestamp: Date.now(), + signature: 'sig1', + appSpecifications: { name: 'enterpriseApp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'encBlob' }, + valueSat: 1e8, txid: 'tx1', height: 2000, + }, + ]; + + // Generate > 500 missing hashes to trigger bulk fetch path + const manyMissing = Array(600).fill(null).map((_, i) => ({ + hash: `hash${i}`, txid: `tx${i}`, height: 1999 + i, value: 1e8, message: false, + })); + + let getMissingCalls = 0; + localDbHelperStub.findInDatabase.callsFake(() => { + getMissingCalls += 1; + if (getMissingCalls === 1) return Promise.resolve(manyMissing); + return Promise.resolve([]); + }); + localDbHelperStub.findOneInDatabase.resolves({ generalScannedHeight: 2555000 }); + + serviceHelperStub.axiosGet.callsFake((url) => { + if (url.includes('permanentmessages')) { + return Promise.resolve(makeStreamResponse(bulkMessages)); + } + return Promise.resolve({ data: { status: 'success', data: true } }); + }); + + // Previous spec is enterprise v8 with encrypted data + const prevSpec = { + name: 'enterpriseApp', version: 8, owner: 'my-owner', enterprise: 'prevEncBlob', + }; + localCollectionStub.find.returns({ + project: sinon.stub().returns({ + sort: sinon.stub().returns({ + toArray: sinon.stub().resolves([ + { + type: 'fluxappregister', hash: 'hash0', height: 1999, + appSpecifications: prevSpec, + }, + ]), + }), + }), + }); + + localMessageVerifierStub.verifyAppMessageUpdateSignature.resolves(true); + + await localModule.syncMissingHashes(); + + // only the update itself is decrypted + sinon.assert.calledOnce(localCheckAndDecryptStub); + const verifyCall = localMessageVerifierStub.verifyAppMessageUpdateSignature.firstCall; + expect(verifyCall.args[5]).to.equal('my-owner'); }); }); diff --git a/tests/unit/appIdentity.test.js b/tests/unit/appIdentity.test.js new file mode 100644 index 0000000000..236f622d6e --- /dev/null +++ b/tests/unit/appIdentity.test.js @@ -0,0 +1,55 @@ +const { expect } = require('chai'); + +const { isValidAppNameAndOwner, verifyAppNameAndOwner } = require('../../ZelBack/src/services/utils/appIdentity'); + +describe('appIdentity tests', () => { + const fluxId = '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC'; + const ethAddress = `0x${'aB3'.repeat(13)}c`; + + describe('isValidAppNameAndOwner tests', () => { + [ + ['a Flux ID owner', 'my-app1', fluxId], + ['an Ethereum owner', 'my-app1', ethAddress], + ['a 63 character name', 'a'.repeat(63), fluxId], + ['a one character name', 'a', fluxId], + ].forEach(([description, name, owner]) => { + it(`should accept ${description}`, () => { + expect(isValidAppNameAndOwner(name, owner)).to.equal(true); + }); + }); + + [ + ['a name with an underscore', 'my_app', fluxId], + ['a name with a space', 'my app', fluxId], + ['a name starting with a hyphen', '-myapp', fluxId], + ['a name ending with a hyphen', 'myapp-', fluxId], + ['a name longer than 63 characters', 'a'.repeat(64), fluxId], + ['an empty name', '', fluxId], + ['a name that is not a string', ['myapp'], fluxId], + ['no name', undefined, fluxId], + ['an owner that is not a Flux ID or Ethereum address', 'myapp', 'my-owner'], + ['an owner with a bad checksum', 'myapp', '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uD'], + ['an Ethereum owner that is too long', 'myapp', `0x${'a'.repeat(41)}`], + ['an owner that is not a string', 'myapp', { id: fluxId }], + ['no owner', 'myapp', undefined], + ].forEach(([description, name, owner]) => { + it(`should refuse ${description}`, () => { + expect(isValidAppNameAndOwner(name, owner)).to.equal(false); + }); + }); + }); + + describe('verifyAppNameAndOwner tests', () => { + it('should return for a valid name and owner', () => { + expect(() => verifyAppNameAndOwner('myapp', fluxId)).to.not.throw(); + }); + + it('should throw for an invalid name', () => { + expect(() => verifyAppNameAndOwner('my_app', fluxId)).to.throw('Invalid app name or owner'); + }); + + it('should throw for an invalid owner', () => { + expect(() => verifyAppNameAndOwner('myapp', 'my-owner')).to.throw('Invalid app name or owner'); + }); + }); +}); diff --git a/tests/unit/appInspector.test.js b/tests/unit/appInspector.test.js index 1420160466..e6bfb8fe2e 100644 --- a/tests/unit/appInspector.test.js +++ b/tests/unit/appInspector.test.js @@ -14,6 +14,7 @@ describe('appInspector tests', () => { let configStub; let globalStateStub; let cpuBurstHelperStub; + let cpuThrottleExemptionStub; let appUtilitiesStub; beforeEach(() => { @@ -38,6 +39,10 @@ describe('appInspector tests', () => { isBurstActive: sinon.stub().resolves(false), }; + cpuThrottleExemptionStub = { + isCpuThrottleExempt: sinon.stub().returns(false), + }; + // Hoisted so a test can drive the disk reading. A partial one is a floor // rather than a total, and what the store does with it is the difference // between a chart and a cliff. @@ -87,6 +92,7 @@ describe('appInspector tests', () => { }, '../utils/appUtilities': appUtilitiesStub, '../utils/cpuBurstHelper': cpuBurstHelperStub, + '../utils/cpuThrottleExemption': cpuThrottleExemptionStub, 'node-cmd': { run: (cmd, callback) => callback(null, 'data', 'stderr'), }, @@ -334,13 +340,7 @@ describe('appInspector tests', () => { expect(stored.memoryCache).to.equal(512); }); - it('takes the cgroup v1 name when the v2 one is absent', async () => { - const stored = await tickOnce({ stats: { cache: 300 } }); - - expect(stored.memoryCache).to.equal(300); - }); - - it('records no cache rather than a zero when the host reports neither', async () => { + it('records no cache rather than a zero when the host does not report it', async () => { // Zero would read as "nothing cached" and be silently subtracted; null says // the figure is unavailable, and leaves the consumer where it was before. const stored = await tickOnce({}); @@ -2359,6 +2359,95 @@ describe('appInspector tests', () => { expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly('db_myapp', 1.8e9)).to.be.true; }); + + // Hub game servers. A game's tick is a latency budget, so the 80% cut shows up as lag, + // and a saturated server measured against its reduced allocation never earns it back. + describe('throttle-exempt apps', () => { + const gameApp = { name: 'enshrouded1792177003381', version: 3, cpu: 3 }; + + beforeEach(() => { + cpuThrottleExemptionStub.isCpuThrottleExempt.callsFake((name) => name === gameApp.name); + }); + + it('never lowers a saturated exempt app', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 3e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.called).to.be.false; + }); + + it('restores an exempt app already lowered to 80%, even while it is saturated', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 2.4e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly(gameApp.name, 3e9)).to.be.true; + }); + + it('restores without waiting for a full window of samples', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 2.4e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: [] } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly(gameApp.name, 3e9)).to.be.true; + }); + + it('consumes the window, so the samples it declined to judge are never held against it', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 3e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]), lastCpuDecisionAt: 0 } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(globalStateStub.appsMonitored[gameApp.name].lastCpuDecisionAt).to.be.above(0); + }); + + it('still throttles an app that is not exempt in the same pass', async () => { + dockerServiceStub.dockerContainerInspect.resolves({ HostConfig: { NanoCpus: 2e9 }, State: { Pid: 1234 } }); + globalStateStub.appsMonitored = { + [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) }, + myapp: { statsStore: window([1, 1, 1, 1, 1]) }, + }; + const both = sinon.stub().resolves({ status: 'success', data: [{ ...gameApp, cpu: 2 }, simpleApp] }); + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, both); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly('myapp', 1.8e9)).to.be.true; + }); + + it('holds every component of an exempt composed app at its own spec', async () => { + const composed = { + name: 'palworld1792177003381', + version: 8, + compose: [{ name: 'game', cpu: 4 }, { name: 'sidecar', cpu: 2 }], + }; + cpuThrottleExemptionStub.isCpuThrottleExempt.callsFake((name) => name === composed.name); + dockerServiceStub.dockerContainerInspect.callsFake(async (name) => ({ + HostConfig: { NanoCpus: name.startsWith('game_') ? 3.2e9 : 2e9 }, + State: { Pid: 1234 }, + })); + globalStateStub.appsMonitored = { + game_palworld1792177003381: { statsStore: window([1, 1, 1, 1, 1]) }, + sidecar_palworld1792177003381: { statsStore: window([1, 1, 1, 1, 1]) }, + }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(composed)); + + expect(dockerServiceStub.appDockerUpdateCpu.calledOnceWithExactly('game_palworld1792177003381', 4e9)).to.be.true; + }); + + it('leaves the container alone when docker cannot inspect it', async () => { + dockerServiceStub.dockerContainerInspect.resolves(null); + globalStateStub.appsMonitored = { [gameApp.name]: { statsStore: window([1, 1, 1, 1, 1]) } }; + + await appInspector.checkApplicationsCpuUSage(globalStateStub.appsMonitored, installedAppsReturning(gameApp)); + + expect(dockerServiceStub.appDockerUpdateCpu.called).to.be.false; + }); + }); }); describe('exported functions', () => { diff --git a/tests/unit/appInstaller.test.js b/tests/unit/appInstaller.test.js index 6014a1306e..87e54500f9 100644 --- a/tests/unit/appInstaller.test.js +++ b/tests/unit/appInstaller.test.js @@ -5,6 +5,15 @@ const { resetGlobalState } = require('./fixtures/globalState'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); + +// what upnpService.mapUpnpPort answers for a port mapped, and for one the router refused +const MAPPED = { + mapped: true, code: null, reason: null, routerWide: false, +}; +const REFUSED = { + mapped: false, code: 728, reason: 'TCP 31000 not mapped: UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)', routerWide: true, +}; // The full-install dockerService stub, shared by every proxyquire setup that // drives registerAppLocally. Pass overrides for the few tests that need a @@ -24,6 +33,13 @@ const makeDockerServiceStub = (overrides = {}) => ({ }); describe('appInstaller tests', () => { + // The placement record is its own module with its own tests; here it is the + // seam an install writes through. Fresh per test. + let appPlacementsStub; + beforeEach(() => { + appPlacementsStub = { recordPlacement: sinon.stub().resolves() }; + }); + let appInstaller; // The same stub map the default instance is built from, so a test that needs one // more seam spreads it rather than restating eighty lines of it. @@ -71,8 +87,6 @@ describe('appInstaller tests', () => { fluxapps: { blocksLasting: 22000, latestAppSpecification: 1, - ownerAppAllowance: 100, - temporaryAppAllowance: 200, maxImageSize: 10000000000, }, }; @@ -98,7 +112,9 @@ describe('appInstaller tests', () => { }; dbHelperStub = { - databaseConnection: sinon.stub(), + // A connection always hands back a client; a test that needs a database + // to answer says what it answers. + databaseConnection: sinon.stub().returns({ db: sinon.stub().returns('database') }), findInDatabase: sinon.stub(), findOneInDatabase: sinon.stub(), insertOneToDatabase: sinon.stub(), @@ -132,6 +148,7 @@ describe('appInstaller tests', () => { // Proxy require baseStubs = { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -139,6 +156,7 @@ describe('appInstaller tests', () => { '../serviceHelper': { ensureString: sinon.stub().returnsArg(0), ensureNumber: sinon.stub().returnsArg(0), + ensureObject: (value) => (typeof value === 'string' ? JSON.parse(value) : value), delay: sinon.stub().resolves(), }, '../generalService': { @@ -207,7 +225,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../utils/globalState': globalStateStub, '../../lib/log': logStub, @@ -377,7 +395,7 @@ describe('appInstaller tests', () => { await appInstaller.installAppLocally(req, res); expect(res.json.calledOnce).to.be.true; - expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.USER, authOf(req))).to.be.true; + expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.FLUX_TEAM, authOf(req))).to.be.true; }); it('should handle missing appname parameter', async () => { @@ -428,9 +446,9 @@ describe('appInstaller tests', () => { }); }); - // Placing a registered app on a node is not the operator's call, for the same - // reason removing one is not. The temporary-message route is deliberately NOT - // closed with it - that is how an app is tested before it is registered. + // Placing an app on a node is not the operator's call, for the same reason + // removing one is not, and an app not yet paid for is never placed: it is + // tried out through the test-install route, which tears it down again. describe('local install is not the node operator\'s to make', () => { const nameInstall = () => ({ params: { appname: 'someCustomerApp' }, query: {} }); @@ -468,27 +486,248 @@ describe('appInstaller tests', () => { expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); }); - // The one that must keep working: an app under test is addressed by HASH and - // carries a temporary message, so it never reaches the by-name gate at all. - it('still lets any logged-in user install an app under test by its temporary message', async () => { - const req = { params: { appname: 'a1b2c3hash' }, query: {} }; + it('refuses a logged-in user installing an app by its temporary message', async () => { + const req = { params: { appname: 'a1b2c3hash' }, query: {}, headers: { zelidauth: JSON.stringify({ zelid: 'someone' }) } }; const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); - verificationHelperStub.verifyPrivilege.withArgs(Privilege.NODE_OPERATOR_OR_FLUX_TEAM, authOf(req)).resolves(false); verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); messageVerifierStub.checkAppTemporaryMessageExistence.resolves({ + hash: 'a1b2c3hash', appSpecifications: { name: 'apptest', version: 3, owner: 'someone' }, }); messageHelperStub.errUnauthorizedMessage.returns({ status: 'error', data: { message: 'Unauthorized' } }); await appInstaller.installAppLocally(req, res); - // It got past the gate: no Unauthorized was returned, and the by-name - // privilege was never consulted because a temporary message was found. - const unauthorized = res.json.getCalls().some((c) => c.args[0]?.data?.message === 'Unauthorized'); - expect(unauthorized, 'a temporary app must not be refused').to.equal(false); - expect(verificationHelperStub.verifyPrivilege.calledWith(Privilege.FLUX_TEAM, authOf(req)), 'the by-name gate must not be reached').to.be.false; + expect(res.json.calledOnce).to.be.true; + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); + expect(dbHelperStub.insertOneToDatabase.called, 'nothing is installed').to.be.false; + }); + + it('does not install a temporary message for the flux team either', async () => { + const req = { params: { appname: 'a1b2c3hash' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + messageVerifierStub.checkAppTemporaryMessageExistence.resolves({ + hash: 'a1b2c3hash', + appSpecifications: { name: 'apptest', version: 3, owner: 'someone' }, + }); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(res.json.calledOnce).to.be.true; + expect(res.json.firstCall.args[0].data.message).to.equal('Application Specifications of a1b2c3hash not found'); + expect(hwRequirementsStub.checkAppHWRequirements.called, 'nothing is installed').to.be.false; + }); + + // An app's term runs from the height its payment was mined at; one without a + // height has no term, and nothing would ever expire it. + it('refuses a specification that carries no height', async () => { + const req = { params: { appname: 'noheightapp' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + baseStubs['../appDatabase/registryManager'].availableApps.resolves([{ name: 'noheightapp', version: 3, owner: 'someone' }]); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(res.json.calledOnce).to.be.true; + expect(res.json.firstCall.args[0].data.message).to.equal('Application Specifications of noheightapp carry no height'); + expect(hwRequirementsStub.checkAppHWRequirements.called, 'nothing is installed').to.be.false; + expect(dbHelperStub.insertOneToDatabase.called, 'nothing is installed').to.be.false; + }); + + it('installs a permanent-lasting local app, whose height is zero', async () => { + const req = { params: { appname: 'foreverapp' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + baseStubs['../appDatabase/registryManager'].availableApps.resolves([{ name: 'foreverapp', version: 3, owner: 'someone', height: 0 }]); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + hwRequirementsStub.checkAppHWRequirements.rejects(new Error('stop here')); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(hwRequirementsStub.checkAppHWRequirements.calledOnce, 'a height of zero was refused').to.be.true; + }); + + it('installs a permanent message with the hash and height the message carries', async () => { + const req = { params: { appname: 'permhash' }, query: {} }; + const res = { json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }; + + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + messageVerifierStub.checkAppMessageExistence.withArgs('permhash').resolves({ + hash: 'permhash', + height: 2000000, + appSpecifications: { name: 'permapp', version: 3, owner: 'someone' }, + }); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findInDatabase.resolves([]); + // stop at the requirement check: the specification it is given is what gets installed + hwRequirementsStub.checkAppHWRequirements.rejects(new Error('stop here')); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + + await appInstaller.installAppLocally(req, res); + + expect(hwRequirementsStub.checkAppHWRequirements.calledOnce).to.be.true; + const installed = hwRequirementsStub.checkAppHWRequirements.firstCall.args[0]; + expect(installed).to.include({ name: 'permapp', hash: 'permhash', height: 2000000 }); + }); + }); + + // An app not yet paid for is tried out by its owner. Anyone else is refused + // before a container is made, and the flux team may test any. + describe('testAppInstall of an app not yet paid for', () => { + const owner = '1OwnerOfTheTemporaryApp'; + const tempMessage = { + hash: 'temphash', + type: 'fluxappregister', + appSpecifications: { + name: 'trialapp', version: 4, owner, compose: [{ name: 'web', repotag: 'org/web:1', cpu: 0.5, ram: 500, hdd: 5 }], + }, + }; + const requestAs = (zelid) => ({ params: { appname: 'temphash' }, query: {}, headers: { zelidauth: JSON.stringify({ zelid, loginPhrase: 'p', signature: 's' }) } }); + const response = () => ({ json: sinon.stub(), setHeader: sinon.stub(), write: sinon.stub(), end: sinon.stub(), writableEnded: false }); + // reached only once the caller has been let through + const pastTheGate = () => baseStubs['../appSystem/systemIntegration'].systemArchitecture.called; + + beforeEach(() => { + messageVerifierStub.checkAppTemporaryMessageExistence.withArgs('temphash').resolves(tempMessage); + messageHelperStub.errUnauthorizedMessage.returns({ status: 'error', data: { message: 'Unauthorized' } }); + // stop at the architecture check: past it the install itself runs + baseStubs['../appSystem/systemIntegration'].systemArchitecture.rejects(new Error('stop here')); + messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); + }); + + it('lets the owner test it', async () => { + const req = requestAs(owner); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(true); + expect(res.json.calledWithMatch({ data: { message: 'Unauthorized' } })).to.equal(false); + }); + + it('refuses another logged-in user', async () => { + const req = requestAs('1SomeoneElse'); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + expect(res.json.calledOnce).to.equal(true); + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); + }); + + it('lets the flux team test it', async () => { + const req = requestAs('1FluxTeam'); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(true); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(true); + }); + + it('refuses to test an app this node holds, before it takes the node', async () => { + const req = requestAs(owner); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findOneInDatabase.resolves({ name: 'trialapp' }); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate(), 'the test went on to install over the app').to.equal(false); + expect(res.json.firstCall.args[0].data.message).to.equal('Flux App trialapp is installed on this node. Test it on a node that does not hold it.'); + }); + + // An update is signed by the app's owner as registered, whatever owner it names. + describe('an update that hands the app to a new owner', () => { + const current = '1CurrentOwner'; + const named = '1NewOwnerNamedInTheUpdate'; + + beforeEach(() => { + messageVerifierStub.checkAppTemporaryMessageExistence.withArgs('temphash').resolves({ + ...tempMessage, type: 'fluxappupdate', appSpecifications: { ...tempMessage.appSpecifications, owner: named }, + }); + baseStubs['../appDatabase/registryManager'].getApplicationGlobalSpecifications + .withArgs('trialapp').resolves({ ...tempMessage.appSpecifications, owner: current }); + }); + + it('lets the current owner test it', async () => { + const req = requestAs(current); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(true); + }); + + it('refuses the owner it names, who does not own the app yet', async () => { + const req = requestAs(named); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); + }); + + it('refuses a session that names no address when the app is not registered', async () => { + baseStubs['../appDatabase/registryManager'].getApplicationGlobalSpecifications.withArgs('trialapp').resolves(null); + const req = requestAs(null); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + }); + + it('refuses everyone but the flux team when the app is not registered', async () => { + baseStubs['../appDatabase/registryManager'].getApplicationGlobalSpecifications.withArgs('trialapp').resolves(null); + const req = requestAs(current); + const res = response(); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.USER, authOf(req)).resolves(true); + verificationHelperStub.verifyPrivilege.withArgs(Privilege.FLUX_TEAM, authOf(req)).resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + }); + }); + + it('refuses a caller with no session at all', async () => { + const req = { params: { appname: 'temphash' }, query: {} }; + const res = response(); + verificationHelperStub.verifyPrivilege.resolves(false); + + await appInstaller.testAppInstall(req, res); + + expect(pastTheGate()).to.equal(false); + expect(res.json.firstCall.args[0].data.message).to.equal('Unauthorized'); }); }); @@ -604,6 +843,7 @@ describe('appInstaller tests', () => { // Mock message verifier to return enterprise app with empty compose messageVerifierStub.checkAppTemporaryMessageExistence.resolves({ + hash: 'enterprisehash', appSpecifications: enterpriseAppSpec, }); @@ -620,8 +860,10 @@ describe('appInstaller tests', () => { } // Verify that decryption was called for enterprise app - expect(enterpriseHelperStub.checkAndDecryptAppSpecs.calledWith(enterpriseAppSpec)).to.be.true; + expect(enterpriseHelperStub.checkAndDecryptAppSpecs.calledWith({ ...enterpriseAppSpec, hash: 'enterprisehash' })).to.be.true; expect(appSpecHelpersStub.specificationFormatter.calledWith(decryptedAppSpec)).to.be.true; + // the formatter's output keeps the hash the message carried + expect(decryptedAppSpec.hash).to.equal('enterprisehash'); expect(logStub.info.calledWith('testAppInstall: enterpriseapp')).to.be.true; }); @@ -667,6 +909,7 @@ describe('appInstaller tests', () => { }; const appInstallerForArchTest = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -781,6 +1024,7 @@ describe('appInstaller tests', () => { }; const appInstallerForArchTest = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -851,7 +1095,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../appRequirements/hwRequirements': hwRequirementsStub, '../appMessaging/messageVerifier': messageVerifierStub, @@ -1000,8 +1244,22 @@ describe('appInstaller tests', () => { expect(globalStateStub.installationInProgress, 'a refusal released someone else\'s hold').to.be.true; }); + it('refuses an app whose removal is pending, taking no hold', async () => { + const res = { write: sinon.stub(), end: sinon.stub() }; + globalStateStub.departingApps.enter(appSpec.name); + try { + const result = await appInstaller.registerAppLocally(appSpec, false, res); + + expect(result).to.equal(InstallOutcome.BUSY); + expect(globalStateStub.installationInProgress, 'the refusal took the install hold').to.be.false; + } finally { + globalStateStub.departingApps.leave(appSpec.name); + } + }); + it('should return false if node tier does not return anything', async () => { const appInstallerWithNodeTier = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1067,7 +1325,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../utils/globalState': globalStateStub, '../../lib/log': logStub, @@ -1116,6 +1374,7 @@ describe('appInstaller tests', () => { // present for that reason, and appQueryService - the first call PAST the guard - rejects, // so a guard that stops matching lands on FAILED rather than running on into the install. const installerReachingTheAlreadyInstalledGuard = () => proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../messageHelper': messageHelperStub, '../dbHelper': { @@ -1185,14 +1444,15 @@ describe('appInstaller tests', () => { // has: the row is written before the image is fetched, so the claim exists by // the time the install gives up. `failBeforeRow` moves the failure ahead of // the row instead. - function buildFailingInstaller(removeAppLocallyStub, { failBeforeRow = false } = {}) { + function buildFailingInstaller(removeAppLocallyStub, { failBeforeRow = false, alreadyInstalled = false } = {}) { const dbHelperStubLocal = { databaseConnection: sinon.stub().returns({ db: () => ({ collection: () => ({}) }) }), findInDatabase: sinon.stub().resolves([]), - // 1st call = "already installed?" -> null so the install proceeds. + // 1st call = "already installed?" -> null so the install proceeds, unless + // the app is already installed here. findOneInDatabase: (() => { const stub = sinon.stub().resolves({ name: 'testapp' }); - stub.onFirstCall().resolves(null); + stub.onFirstCall().resolves(alreadyInstalled ? { name: 'testapp' } : null); return stub; })(), findOneAndDeleteInDatabase: sinon.stub().resolves(), @@ -1200,6 +1460,7 @@ describe('appInstaller tests', () => { }; return proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1273,7 +1534,7 @@ describe('appInstaller tests', () => { }, '../upnpService': { isUPNP: sinon.stub().returns(false), - mapUpnpPort: sinon.stub().resolves(true), + mapUpnpPort: sinon.stub().resolves(MAPPED), }, '../utils/globalState': globalStateStub, '../../lib/log': logStub, @@ -1358,6 +1619,27 @@ describe('appInstaller tests', () => { expect(removeAppLocallyStub.firstCall.args[4], 'broadcast a removal for an app this node never had').to.equal(false); }); + // From its row on, the app is installed here and a peer may list this node as a + // holder: until the monitor has measured the folder, a peer about to seed waits. + it('records the app\'s receive-only folder as not looked at once its row is written', async () => { + folderStateMap.clear(); + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves()); + + await appInstallerWithDb.registerAppLocally({ ...appSpec, name: 'fluxinst', containerData: 'g:/appdata' }, false, { write: sinon.stub(), end: sinon.stub() }); + + expect(folderStateMap.get('fluxinst')).to.deep.equal({ state: 'notLookedAt' }); + folderStateMap.clear(); + }); + + it('records nothing when it gave up before the app reached the table', async () => { + folderStateMap.clear(); + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves(), { failBeforeRow: true }); + + await appInstallerWithDb.registerAppLocally({ ...appSpec, name: 'fluxinst', containerData: 'g:/appdata' }, false, { write: sinon.stub(), end: sinon.stub() }); + + expect(folderStateMap.has('fluxinst')).to.equal(false); + }); + // A test install writes the app's row like any other and throws it away, so the // announcement must not read that row as a claim. The mark outlives the teardown // that deletes the row: released before it, a cycle in between claims the app. @@ -1402,6 +1684,47 @@ describe('appInstaller tests', () => { ).to.be.false; }); + // When this node took the app is written with the app's row; a redeploy's + // reinstall finds the record it kept and leaves it alone. + describe('the placement record', () => { + it('is written with the app\'s row', async () => { + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves()); + + await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, false, true); + + sinon.assert.calledOnceWithExactly(appPlacementsStub.recordPlacement, 'testapp'); + }); + + it('is not written for a test install', async () => { + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves()); + + await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, true); + + sinon.assert.notCalled(appPlacementsStub.recordPlacement); + }); + + it('is not written when the install gave up before the app\'s row', async () => { + const appInstallerWithDb = buildFailingInstaller(sinon.stub().resolves(), { failBeforeRow: true }); + + await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, false, true); + + sinon.assert.notCalled(appPlacementsStub.recordPlacement); + }); + }); + + // A test install refused because the app is already installed here created + // nothing, and the installed app is not its to remove. + it('leaves an app already installed here alone when a test install of it is refused', async () => { + const removeAppLocallyStub = sinon.stub().resolves(); + const appInstallerWithDb = buildFailingInstaller(removeAppLocallyStub, { alreadyInstalled: true }); + + const outcome = await appInstallerWithDb.registerAppLocally(appSpec, false, { write: sinon.stub(), end: sinon.stub() }, true); + + expect(outcome, 'never reached the refusal this test is about').to.equal(InstallOutcome.ALREADY_INSTALLED); + expect(removeAppLocallyStub.called, 'a refused test install removed the app installed here').to.equal(false); + expect(globalStateStub.testInstallingApps.has('testapp'), 'left the mark behind').to.equal(false); + }); + it('runs the post-install broadcast only AFTER releasing the install lock', async () => { // The announcement runs for as long as a broadcast cycle takes, and every // other install, removal and redeploy on this node refuses while the install @@ -1426,6 +1749,7 @@ describe('appInstaller tests', () => { }; const appInstallerSuccess = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1479,7 +1803,7 @@ describe('appInstaller tests', () => { }), }, '../pgpService': { decryptMessage: sinon.stub().resolves('user:token') }, - '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(true) }, + '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(MAPPED) }, '../utils/globalState': globalStateStub, '../utils/volumeService': { verifyAppVolumeMount: sinon.stub().resolves(), ensureMountPathsExist: sinon.stub().resolves() }, '../../lib/log': logStub, @@ -1565,7 +1889,7 @@ describe('appInstaller tests', () => { '../appManagement/appInspector': { startAppMonitoring: sinon.stub() }, '../utils/imageVerifier': { ImageVerifier: sinon.stub().returns({ addCredentials: sinon.stub(), verifyImage: sinon.stub().resolves(), throwIfError: sinon.stub(), supported: true, provider: 'docker.io' }) }, '../pgpService': { decryptMessage: sinon.stub().resolves('user:token') }, - '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(true) }, + '../upnpService': { isUPNP: sinon.stub().returns(false), mapUpnpPort: sinon.stub().resolves(MAPPED) }, '../utils/enterpriseHelper': enterpriseHelperStub, '../utils/appSpecHelpers': appSpecHelpersStub, '../../lib/log': logStub, @@ -1610,9 +1934,10 @@ describe('appInstaller tests', () => { allowPort: sinon.stub().resolves({ status: true }), allowAppPorts: sinon.stub().resolves({ failed: [], locked: false }), }; - upnpStub = { isUPNP: sinon.stub().returns(true), mapUpnpPort: sinon.stub().resolves(true) }; + upnpStub = { isUPNP: sinon.stub().returns(true), mapUpnpPort: sinon.stub().resolves(MAPPED) }; logStub = { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }; appInstallerPorts = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../fluxNetworkHelper': firewallStub, '../upnpService': upnpStub, '../../lib/log': logStub, @@ -1649,6 +1974,22 @@ describe('appInstaller tests', () => { expect(upnpStub.mapUpnpPort.calledWith(31000, 'Flux_App_myapp')).to.be.true; }); + it('fails the install with the router\'s reason when a port is refused', async () => { + upnpStub.mapUpnpPort.resolves(REFUSED); + + const error = await appInstallerPorts.setupApplicationPorts({ ports: [31000] }, 'myapp', false, null).catch((e) => e); + + expect(error.message).to.equal(`Error: Port 31000 FAILed to map. ${REFUSED.reason}`); + }); + + it('fails a v1 install with the router\'s reason when its port is refused', async () => { + upnpStub.mapUpnpPort.resolves(REFUSED); + + const error = await appInstallerPorts.setupApplicationPorts({ port: 31000 }, 'myapp', false, null).catch((e) => e); + + expect(error.message).to.equal(`Error: Port 31000 FAILed to map. ${REFUSED.reason}`); + }); + it('opens nothing for a test install', async () => { await appInstallerPorts.setupApplicationPorts({ ports: [31000] }, 'myapp', false, null, true); @@ -1666,6 +2007,7 @@ describe('appInstaller tests', () => { dockerServiceStub = makeDockerServiceStub({ getFreeFluxAppNetworkOctet: sinon.stub().resolves(7) }); egressRulesStub = sinon.stub().resolves(true); appInstallerNet = proxyquire('../../ZelBack/src/services/appLifecycle/appInstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../serviceHelper': { ensureString: sinon.stub().returnsArg(0) }, '../dockerService': dockerServiceStub, '../fluxNetworkHelper': { applyContainerEgressRules: egressRulesStub }, diff --git a/tests/unit/appKill.test.js b/tests/unit/appKill.test.js new file mode 100644 index 0000000000..3c0b1b9d51 --- /dev/null +++ b/tests/unit/appKill.test.js @@ -0,0 +1,303 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const appKill = require('../../ZelBack/src/services/appLifecycle/appKill'); + +describe('appKill tests', () => { + let root; + let cgroupRoot; + let procRoot; + let dockerRoot; + let killed; + // unit -> its cgroup, as systemd reports it + let units; + + const id = (n) => String(n).repeat(64).slice(0, 64); + // systemd runs in a container here, so its tree starts below the filesystem's root + const SYSTEMD_ROOT = '/init'; + + function cgroup(rel, pids) { + const dir = path.join(cgroupRoot, SYSTEMD_ROOT, rel); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'cgroup.procs'), pids.map((pid) => `${pid}\n`).join('')); + fs.writeFileSync(path.join(dir, 'cgroup.kill'), ''); + units.set(path.basename(rel), path.join(SYSTEMD_ROOT, rel)); + const parent = rel.split('/')[0]; + if (parent !== rel) units.set(parent, path.join(SYSTEMD_ROOT, parent)); + return dir; + } + // `app`: the runonflux.app label FluxOS sets; left out, a container made before it + function record(n, name, pid = 0, { app } = {}) { + fs.mkdirSync(path.join(dockerRoot, 'containers', id(n)), { recursive: true }); + const labels = app ? { 'runonflux.app': app } : {}; + fs.writeFileSync(path.join(dockerRoot, 'containers', id(n), 'config.v2.json'), JSON.stringify({ Name: `/${name}`, State: { Pid: Number(pid) }, Config: { Labels: labels } })); + } + function container(n, name, pids) { + record(n, name, pids[0]); + return cgroup(path.join('system.slice', `docker-${id(n)}.scope`), pids); + } + // An app's slice, as the kernel lays it out: no process of its own, and each + // container's processes in the container's scope below it. + function slice(unit, scopes) { + const dir = cgroup(path.join('flux.slice', unit), []); + Object.entries(scopes).forEach(([scope, pids]) => cgroup(path.join('flux.slice', unit, scope), pids)); + return dir; + } + function processWith(pid, { sigPnd = '0000000000000000', shdPnd = '0000000000000000' } = {}) { + fs.mkdirSync(path.join(procRoot, pid), { recursive: true }); + fs.writeFileSync(path.join(procRoot, pid, 'status'), `Name:\tapp\nSigPnd:\t${sigPnd}\nShdPnd:\t${shdPnd}\n`); + } + + // what sudo would do: read and list as root, and write to cgroup.kill; and + // what systemd answers about a unit's cgroup + function asRoot(cmd, { params }) { + if (cmd === 'systemctl') return Promise.resolve({ stdout: `${units.get(params[4]) ?? ''}\n`, error: null }); + if (cmd === 'ls') return Promise.resolve({ stdout: fs.readdirSync(params[1]).join('\n'), error: null }); + if (cmd === 'cat') { + try { + return Promise.resolve({ stdout: fs.readFileSync(params[0], 'utf8'), error: null }); + } catch (error) { + return Promise.resolve({ stdout: '', error }); + } + } + if (cmd === 'sh') { + killed.push(path.dirname(params[3])); + return Promise.resolve({ stdout: '', error: null }); + } + return Promise.resolve({ stdout: '', error: new Error(`unexpected ${cmd}`) }); + } + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'appkill-')); + cgroupRoot = path.join(root, 'cgroup'); + procRoot = path.join(root, 'proc'); + dockerRoot = path.join(root, 'docker'); + fs.mkdirSync(path.join(dockerRoot, 'containers'), { recursive: true }); + appKill.usePaths({ cgroupRoot, procRoot, dockerRoot }); + killed = []; + units = new Map(); + sinon.stub(serviceHelper, 'runCommand').callsFake(asRoot); + }); + + afterEach(() => { + appKill.reset(); + fs.rmSync(root, { recursive: true, force: true }); + sinon.restore(); + }); + + describe('kill', () => { + it('kills the app\'s slice and the scopes of its containers made before slices, and no other app\'s', async () => { + const appSliceDir = slice('flux-my\\x2dapp.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + record(9, 'fluxdb_my-app', '10'); + const web = container(1, 'fluxweb_my-app', ['11']); + const single = container(2, 'fluxmy-app', ['12']); + container(3, 'fluxweb_my-app2', ['13']); + container(4, 'fluxweb_other', ['14']); + slice('flux-my.slice', { [`docker-${id(8)}.scope`]: ['15'] }); + + const sent = await appKill.kill('my-app', await appKill.readContainers()); + + expect(sent).to.have.members([appSliceDir, web, single]); + expect(killed).to.have.members([appSliceDir, web, single]); + }); + + it('sends nothing for an app with no cgroup left', async () => { + fs.mkdirSync(path.join(dockerRoot, 'containers', id(5)), { recursive: true }); + fs.writeFileSync(path.join(dockerRoot, 'containers', id(5), 'config.v2.json'), JSON.stringify({ Name: '/fluxgone' })); + + expect(await appKill.kill('gone', await appKill.readContainers())).to.deep.equal([]); + expect(killed).to.deep.equal([]); + }); + + it('kills the app\'s slice alone when docker\'s records could not be read', async () => { + const appSliceDir = slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + container(1, 'fluxweb_app', ['11']); + + expect(await appKill.kill('app', null)).to.deep.equal([appSliceDir]); + }); + }); + + describe('killOutsideSlices', () => { + it('kills every container outside the app slices, and no other unit', async () => { + slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + const web = container(1, 'fluxweb_app', ['11']); + const other = container(2, 'fluxother', ['12']); + cgroup('system.slice/fluxos.service', ['1']); + + expect(await appKill.killOutsideSlices()).to.have.members([web, other]); + expect(killed).to.have.members([web, other]); + }); + + it('kills nothing and says so when systemd does not say where system.slice is', async () => { + slice('flux-app.slice', { [`docker-${id(9)}.scope`]: ['10'] }); + + expect(await appKill.killOutsideSlices()).to.equal(null); + expect(killed).to.deep.equal([]); + }); + + it('passes over a container record it cannot read', async () => { + const web = container(1, 'fluxweb_app', ['11']); + fs.mkdirSync(path.join(dockerRoot, 'containers', id(6)), { recursive: true }); + fs.writeFileSync(path.join(dockerRoot, 'containers', id(6), 'config.v2.json'), '{ broken'); + + expect(await appKill.kill('app', await appKill.readContainers())).to.deep.equal([web]); + }); + }); + + describe('stillRunnable', () => { + beforeEach(() => { + slice('flux-app.slice', { [`docker-${id(1)}.scope`]: ['21', '22'], [`docker-${id(2)}.scope`]: ['23', '24'] }); + }); + + it('names each process that is neither gone nor has the kill pending', async () => { + processWith('21'); + processWith('22', { shdPnd: '0000000000000100' }); + processWith('23', { sigPnd: '0000000000004100' }); + // 24 is gone + + expect(await appKill.stillRunnable('app', [])).to.deep.equal(['21']); + }); + + it('does not take another pending signal for the kill', async () => { + ['21', '22', '23', '24'].forEach((pid) => processWith(pid, { shdPnd: '0000000000004000' })); + + expect(await appKill.stillRunnable('app', [])).to.deep.equal(['21', '22', '23', '24']); + }); + + it('reads the processes in every scope under the slice, the slice itself holding none', async () => { + ['21', '22', '23', '24'].forEach((pid) => processWith(pid)); + + expect(await appKill.stillRunnable('app', [])).to.have.members(['21', '22', '23', '24']); + }); + + it('names nothing for an app with no cgroup left', async () => { + expect(await appKill.stillRunnable('other', [])).to.deep.equal([]); + }); + + describe('when docker\'s records could not be read', () => { + it('names what of the app\'s slice can still run', async () => { + processWith('21'); + container(3, 'fluxother_other', ['31']); + + expect(await appKill.stillRunnable('app', null)).to.deep.equal(['21']); + }); + + it('cannot say the app can no longer run while a container outside the slices, which may be the app\'s, can run', async () => { + container(3, 'fluxweb_other', ['31']); + processWith('31'); + + expect(await appKill.stillRunnable('app', null)).to.equal(null); + }); + + it('says the app can no longer run once every container outside the slices has the kill pending', async () => { + container(3, 'fluxweb_other', ['31']); + processWith('31', { sigPnd: '0000000000000100' }); + + expect(await appKill.stillRunnable('app', null)).to.deep.equal([]); + }); + + it('says the app can no longer run when every container lives in a slice', async () => { + cgroup('system.slice/fluxos.service', ['1']); + + expect(await appKill.stillRunnable('app', null)).to.deep.equal([]); + }); + + it('cannot say the app can no longer run when systemd does not say where system.slice is', async () => { + expect(await appKill.stillRunnable('app', null)).to.equal(null); + }); + }); + }); + + describe('stillRunnableOutsideSlices', () => { + it('names what can still run of every container outside the app slices', async () => { + container(1, 'fluxweb_a', ['11']); + container(2, 'fluxweb_b', ['12', '13']); + cgroup('system.slice/fluxos.service', ['1']); + ['1', '11', '12'].forEach((pid) => processWith(pid)); + processWith('13', { sigPnd: '0000000000000100' }); + + expect(await appKill.stillRunnableOutsideSlices()).to.have.members(['11', '12']); + }); + + it('cannot say when systemd does not say where system.slice is', async () => { + expect(await appKill.stillRunnableOutsideSlices()).to.equal(null); + }); + }); + + describe('readContainers', () => { + it('answers null when docker\'s records cannot be listed', async () => { + fs.rmSync(path.join(dockerRoot, 'containers'), { recursive: true }); + serviceHelper.runCommand.withArgs('ls').resolves({ stdout: '', error: new Error('No such file or directory') }); + + expect(await appKill.readContainers()).to.equal(null); + }); + }); + + describe('appsOnDisk', () => { + it('names every app with a slice or a container record, and no container that is not an app\'s', async () => { + slice('flux-sliced\\x2dapp.slice', {}); + container(1, 'fluxweb_composed', ['11']); + container(2, 'fluxsingle', ['12']); + container(3, 'watchtower', ['13']); + + expect(await appKill.appsOnDisk(await appKill.readContainers())).to.have.members(['sliced-app', 'composed', 'single']); + }); + + it('names every app with a slice when docker\'s records could not be read', async () => { + slice('flux-sliced.slice', {}); + container(1, 'fluxweb_composed', ['11']); + + expect(await appKill.appsOnDisk(null)).to.deep.equal(['sliced']); + }); + + it('names a labelled container\'s app by its label, and a zel component\'s by its name', async () => { + record(1, 'fluxgame', 0, { app: 'fluxgame' }); + record(2, 'zelcash_legacy'); + + expect(await appKill.appsOnDisk(await appKill.readContainers())).to.have.members(['fluxgame', 'legacy']); + }); + }); + + describe('containersOf', () => { + it('names every container of the app from docker\'s records, and no other app\'s', async () => { + record(1, 'fluxweb_composed'); + record(2, 'fluxdb_composed'); + record(3, 'fluxweb_composedtoo'); + record(4, 'fluxcomposed'); + + expect(appKill.containersOf('composed', await appKill.readContainers())).to.have.members(['fluxweb_composed', 'fluxdb_composed', 'fluxcomposed']); + }); + + it('takes a container\'s label over its name', async () => { + record(1, 'fluxweb_composed', 0, { app: 'other' }); + record(2, 'renamed', 0, { app: 'composed' }); + + expect(appKill.containersOf('composed', await appKill.readContainers())).to.have.members(['renamed']); + }); + + it('finds a label-less component whose name starts with zel', async () => { + record(1, 'zelcash_legacy'); + + expect(appKill.containersOf('legacy', await appKill.readContainers())).to.have.members(['zelcash_legacy']); + }); + + it('finds the label-less container of an app whose own name starts with flux', async () => { + record(1, 'fluxgame'); + + expect(appKill.containersOf('fluxgame', await appKill.readContainers())).to.have.members(['fluxgame']); + }); + + it('names none when docker\'s records could not be read', () => { + expect(appKill.containersOf('composed', null)).to.deep.equal([]); + }); + + it('does not take a container FluxOS did not name for an app whose name it ends with', async () => { + record(1, 'my_composed'); + + expect(appKill.containersOf('composed', await appKill.readContainers())).to.deep.equal([]); + }); + }); +}); diff --git a/tests/unit/appNetworkLinker.test.js b/tests/unit/appNetworkLinker.test.js index 51b82d838f..f0f87b7669 100644 --- a/tests/unit/appNetworkLinker.test.js +++ b/tests/unit/appNetworkLinker.test.js @@ -175,9 +175,17 @@ describe('appNetworkLinker tests', () => { expect(dockerServiceStub.appDockerNetworkConnect.calledWith('fluxweb_appC')).to.equal(false); }); - it('does not throw when the database read fails', async () => { + it('does not throw when the database read fails, and reconnects once a later read succeeds', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); dbHelperStub.findInDatabase.rejects(new Error('db down')); await expect(appNetworkLinker.reconnectLinkedApps('appA')).to.not.be.rejected; + await expect(appNetworkLinker.reconnectLinkedApps('appA')).to.not.be.rejected; + + dbHelperStub.findInDatabase.resolves([{ name: 'appB', description: 'networkWith:[appA]' }]); + dockerServiceStub.getAppContainerNames.withArgs('appB').resolves(['fluxweb_appB']); + await clock.tickAsync(30 * 1000); + + sinon.assert.calledOnceWithExactly(dockerServiceStub.appDockerNetworkConnect, 'fluxweb_appB', 'fluxDockerNetwork_appA'); }); }); @@ -277,9 +285,23 @@ describe('appNetworkLinker tests', () => { sinon.assert.calledWith(dockerServiceStub.appDockerNetworkConnect, 'fluxweb_appB', 'fluxDockerNetwork_appA'); }); - it('does not throw when the database read fails', async () => { + it('does not throw when the database read fails, and retries with a growing delay until a read succeeds', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); dbHelperStub.findInDatabase.rejects(new Error('db down')); await expect(appNetworkLinker.reconcileAllAppNetworkLinks()).to.not.be.rejected; + + await clock.tickAsync(30 * 1000); + expect(dbHelperStub.findInDatabase.callCount).to.equal(2); + await clock.tickAsync(30 * 1000); + expect(dbHelperStub.findInDatabase.callCount, 'the second retry waits 60 s').to.equal(2); + + dbHelperStub.findInDatabase.resolves([{ name: 'appB', description: 'networkWith:[appA]' }]); + dockerServiceStub.getAppContainerNames.withArgs('appB').resolves(['fluxweb_appB']); + await clock.tickAsync(30 * 1000); + + sinon.assert.calledOnceWithExactly(dockerServiceStub.appDockerNetworkConnect, 'fluxweb_appB', 'fluxDockerNetwork_appA'); + await clock.tickAsync(60 * 60 * 1000); + expect(dbHelperStub.findInDatabase.callCount, 'nothing runs after a read succeeds').to.equal(3); }); }); }); diff --git a/tests/unit/appPlacements.test.js b/tests/unit/appPlacements.test.js new file mode 100644 index 0000000000..42e1ee813c --- /dev/null +++ b/tests/unit/appPlacements.test.js @@ -0,0 +1,189 @@ +const { expect } = require('chai'); +const config = require('config'); +const proxyquire = require('proxyquire'); +const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const appPlacements = require('../../ZelBack/src/services/appManagement/appPlacements'); +const { requireMongo } = require('./dbTestHelper'); + +describe('appPlacements', () => { + before(requireMongo); + + const me = '203.0.113.50:16127'; + const other = '203.0.113.51:16127'; + const june = new Date('2026-06-01T00:00:00.000Z'); + const july = new Date('2026-07-01T00:00:00.000Z'); + let placements; + let locations; + + const recorded = async () => (await placements.find({}, { projection: { _id: 0 } }).sort({ name: 1 }).toArray()); + + beforeEach(async () => { + placements = dbHelper.databaseConnection().db(config.database.appslocal.database) + .collection(config.database.appslocal.collections.appsPlacements); + locations = dbHelper.databaseConnection().db(config.database.appsglobal.database) + .collection(config.database.appsglobal.collections.appsLocations); + await placements.deleteMany({}); + await locations.deleteMany({ ip: { $in: [me, other] } }); + await appPlacements.prepareCollection(); + }); + + afterEach(async () => { + await placements.deleteMany({}); + await locations.deleteMany({ ip: { $in: [me, other] } }); + }); + + it('holds one record per app name', async () => { + const indexes = await placements.indexes(); + expect(indexes.find((index) => index.name === 'name_unique')).to.include({ unique: true }); + }); + + // A record written as an install writes it: dated at the call, over whatever + // record the app had. + const seed = (name, placedAt) => placements.insertOne({ name, placedAt }); + + describe('recordPlacement', () => { + it('records that this node took an app now', async () => { + const before = Date.now(); + await appPlacements.recordPlacement('web'); + + const [record] = await recorded(); + expect(record.name).to.equal('web'); + expect(record.placedAt.getTime()).to.be.within(before, Date.now()); + }); + + // A hard install creates the app's volume: the record of the data before it + // describes data this node no longer holds. + it('replaces a record the app already had', async () => { + await seed('web', june); + const before = Date.now(); + + await appPlacements.recordPlacement('web'); + + const records = await recorded(); + expect(records).to.have.length(1); + expect(records[0].placedAt.getTime()).to.be.within(before, Date.now()); + }); + + it('keeps one record when two writers race the first one', async () => { + await Promise.all([ + appPlacements.recordPlacement('web'), + appPlacements.recordPlacement('web'), + appPlacements.recordPlacement('web'), + ]); + + expect(await recorded()).to.have.length(1); + }); + }); + + describe('placementsOf', () => { + it('answers the apps it has records for, and only those', async () => { + await seed('web', june); + await seed('db', july); + + const found = await appPlacements.placementsOf(['web', 'cache']); + + expect([...found]).to.deep.equal([['web', june]]); + }); + }); + + describe('removePlacement', () => { + it('forgets the one app', async () => { + await seed('web', june); + await seed('db', july); + + await appPlacements.removePlacement('web'); + + expect(await recorded()).to.deep.equal([{ name: 'db', placedAt: july }]); + }); + }); + + describe('removeStrayPlacements', () => { + it('removes the record of an app not installed here, and only that', async () => { + await seed('gone', june); + await seed('web', july); + + const removed = await appPlacements.removeStrayPlacements(['web']); + + expect(removed).to.equal(1); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: july }]); + }); + + it('removes every record when nothing is installed', async () => { + await seed('gone', june); + + await appPlacements.removeStrayPlacements([]); + + expect(await recorded()).to.deep.equal([]); + }); + }); + + describe('backfillPlacements', () => { + const row = (name, ip, runningSince) => ({ + name, ip, hash: `${name}hash`, broadcastedAt: new Date(), expireAt: new Date(Date.now() + 3600000), runningSince, + }); + + // Seeding from the row keeps every app where the network already ranks it, so + // a node that has just upgraded does not become the newest copy of all it holds. + it('dates an installed app with no record at the runningSince of this node\'s own row', async () => { + await locations.insertOne(row('web', me, june)); + + const result = await appPlacements.backfillPlacements(['web'], me); + + expect(result).to.deep.equal({ created: 1, fromLocationRows: 1 }); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + }); + + it('reads only this node\'s row, never another node\'s', async () => { + await locations.insertOne(row('web', other, june)); + const before = Date.now(); + + await appPlacements.backfillPlacements(['web'], me); + + const [{ placedAt }] = await recorded(); + expect(placedAt.getTime()).to.be.within(before, Date.now()); + }); + + it('dates an installed app with neither a record nor a row now', async () => { + const before = Date.now(); + + await appPlacements.backfillPlacements(['web'], me); + + const [{ placedAt }] = await recorded(); + expect(placedAt.getTime()).to.be.within(before, Date.now()); + }); + + it('leaves a record it already has, whatever the row says', async () => { + await seed('web', june); + await locations.insertOne(row('web', me, july)); + + const result = await appPlacements.backfillPlacements(['web'], me); + + expect(result).to.deep.equal({ created: 0, fromLocationRows: 0 }); + expect(await recorded()).to.deep.equal([{ name: 'web', placedAt: june }]); + }); + + it('touches no record of an app not installed here', async () => { + await seed('gone', june); + + await appPlacements.backfillPlacements(['web'], me); + + expect((await recorded()).map((r) => r.name)).to.deep.equal(['gone', 'web']); + }); + }); + + describe('waitForBackfill', () => { + it('resolves once the backfill has written, and not before', async () => { + const fresh = proxyquire('../../ZelBack/src/services/appManagement/appPlacements', {}); + let resolved = false; + const waiting = fresh.waitForBackfill().then(() => { resolved = true; }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(resolved, 'resolved before the backfill ran').to.equal(false); + + await fresh.backfillPlacements(['web'], me); + await waiting; + + expect(resolved).to.equal(true); + expect((await recorded()).map((r) => r.name)).to.deep.equal(['web']); + }); + }); +}); diff --git a/tests/unit/appQueryService.test.js b/tests/unit/appQueryService.test.js index 8bb79db4d1..55faec2d93 100644 --- a/tests/unit/appQueryService.test.js +++ b/tests/unit/appQueryService.test.js @@ -287,13 +287,17 @@ describe('appQueryService tests', () => { expect(await appQueryService.installedAppsListedAt('198.51.100.1:16127')).to.deep.equal(['kept']); }); - it('names none when the installed apps cannot be read', async () => { + it('rejects when the installed apps cannot be read', async () => { dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('appsDatabase') }); dbHelperStub.findInDatabase.rejects(new Error('db down')); - messageHelperStub.createErrorMessage.returns({ status: 'error' }); + messageHelperStub.createErrorMessage.returns({ status: 'error', data: { message: 'db down' } }); registryManagerStub.appsListedAt.resolves(['kept']); - expect(await appQueryService.installedAppsListedAt('198.51.100.1:16127')).to.deep.equal([]); + const error = await appQueryService.installedAppsListedAt('198.51.100.1:16127').catch((caught) => caught); + + expect(error).to.be.an('error'); + expect(error.message).to.include('db down'); + sinon.assert.notCalled(registryManagerStub.appsListedAt); }); }); @@ -601,11 +605,32 @@ describe('appQueryService tests', () => { }); }); + // The apps installed on this node, as the local database answers them. + const installedAppsAre = (apps) => { + dbHelperStub.databaseConnection.returns({ db: () => ({}) }); + dbHelperStub.findInDatabase.resolves(apps); + }; + // Two g: components and one that runs on every instance; the anchor is db. + const twoElected = { + name: 'App', + version: 7, + compose: [ + { name: 'cache', containerData: '/cache' }, + { name: 'db', containerData: 'g:/data' }, + { name: 'web', containerData: 'g:/srv' }, + ], + }; + const oneElected = { name: 'Solo', version: 7, compose: [{ name: 'game', containerData: 'g:/data' }, { name: 'web', containerData: '/www' }] }; + describe('heldComponents', () => { // What a peer mid-election is told this node owns. Answering short here is not // a stale reading - it is a second container started on a volume this node is // already writing, which corrupts it. + beforeEach(() => { + installedAppsAre([]); + }); + // A primary back from a stop: its folder still sends, paused, and nothing runs // or is committed while it decides whether another holder took over. it('reports a component whose folder sends here, with nothing running, committed or stopped', async () => { @@ -645,8 +670,9 @@ describe('appQueryService tests', () => { sinon.stub(appReconciler, 'committedIdentifiers').returns(committed); sinon.stub(primaryRoleChanges, 'promotingIdentifiers').returns(promoting); sinon.stub(appsRuntimeState, 'operatorHeldIdentifiers').resolves(stopped); - messageHelperStub.createDataMessage.returnsArg(0); - return appQueryService.heldComponents(); + messageHelperStub.createDataMessage.callsFake((data) => ({ status: 'success', data })); + const answer = await appQueryService.heldComponents(); + return answer?.status === 'success' ? answer.data : answer; }; // A node becoming the primary has committed before its folder sends and long @@ -725,6 +751,62 @@ describe('appQueryService tests', () => { expect(result).to.deep.equal({ status: 'error' }); expect(messageHelperStub.createDataMessage.called, 'answered with a list built from a failed read').to.be.false; }); + + // A release that elects one g: component asks a peer only about the app's + // anchor. To it a node holding another of the app's g: components holds + // nothing, and it starts the app beside that node. + describe('an app with more than one g: component', () => { + beforeEach(() => { + installedAppsAre([twoElected, oneElected]); + }); + + it('reports the anchor of an app it runs a later g: component of', async () => { + const result = await held({ running: ['fluxweb_App'] }); + + expect(result).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + + it('reports the anchor of an app whose later g: component\'s folder alone sends here', async () => { + syncthingServiceStub.getConfigFolders.resolves([{ id: 'fluxweb_App', type: 'sendreceive' }]); + + const result = await held(); + + expect(result).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + + it('reports the anchor of an app it is becoming the primary of', async () => { + const result = await held({ promoting: ['web_App'] }); + + expect(result).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + + // The non-elected component runs on every instance: holding it is not + // holding the app. + it('adds nothing for a component that is not elected', async () => { + const result = await held({ running: ['fluxcache_App'] }); + + expect(result).to.deep.equal(['fluxcache_App']); + }); + + it('adds nothing for an app with one g: component', async () => { + const result = await held({ running: ['fluxweb_Solo'] }); + + expect(result).to.deep.equal(['fluxweb_Solo']); + }); + + it('adds nothing when it holds none of the app', async () => { + expect(await held()).to.deep.equal([]); + }); + }); + + it('fails the request when the installed apps cannot be read, rather than answering without the anchors', async () => { + dbHelperStub.findInDatabase.rejects(new Error('no primary available')); + messageHelperStub.createErrorMessage.returns({ status: 'error' }); + + const result = await held({ running: ['fluxweb_App'] }); + + expect(result).to.deep.equal({ status: 'error' }); + }); }); // The same account as a peer asks it: sealed - signed over the caller's @@ -736,6 +818,7 @@ describe('appQueryService tests', () => { const sealWith = (seal) => peerIdentityServiceStub.answerSealer.returns(seal); beforeEach(() => { + installedAppsAre([]); nowNs += 10n ** 10n; sinon.stub(process.hrtime, 'bigint').callsFake(() => nowNs); dockerServiceStub.dockerListContainers.resolves([{ Names: ['/fluxwww_App'] }]); @@ -755,6 +838,28 @@ describe('appQueryService tests', () => { sinon.assert.calledOnceWithExactly(peerIdentityServiceStub.answerSealer, 'held', req.body); }); + // A peer asking whether this node has DECIDED it holds the app reads the + // anchor there too, and only where a g: component of the app is decided. + it('carries the anchor in each list only where that list holds the app', async () => { + installedAppsAre([twoElected]); + dockerServiceStub.dockerListContainers.resolves([]); + syncthingServiceStub.getConfigFolders.resolves([{ id: 'fluxweb_App', type: 'sendreceive' }]); + + const answer = await appQueryService.heldComponentsAnswer({ body: {} }, res()); + + expect(answer.data.held).to.have.members(['fluxweb_App', 'fluxdb_App']); + expect(answer.data.decided).to.deep.equal([]); + }); + + it('carries the anchor as decided when a later g: component is decided here', async () => { + installedAppsAre([twoElected]); + dockerServiceStub.dockerListContainers.resolves([{ Names: ['/fluxweb_App'] }]); + + const answer = await appQueryService.heldComponentsAnswer({ body: {} }, res()); + + expect(answer.data.decided).to.have.members(['fluxweb_App', 'fluxdb_App']); + }); + it('holds a component whose folder alone sends here without having decided it', async () => { syncthingServiceStub.getConfigFolders.resolves([{ id: 'fluxgame_App', type: 'sendreceive' }]); @@ -919,13 +1024,16 @@ describe('appQueryService tests', () => { }); beforeEach(() => { - globalState.promotedFolderIds = new Set(['fluxa_a']); - globalState.folderHoldings = new Map([['fluxa_a', { bytes: 5821604997, newestModified: 200 }]]); + globalState.promotedFolderIds = new Set(['fluxw_w']); + globalState.folderStates.set('fluxa_a', { state: 'measured', bytes: 5821604997, newestModified: 200 }); messageHelperStub.createDataMessage.returnsArg(0); + // no app installed unless a test says otherwise + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('appsDatabase') }); + dbHelperStub.findInDatabase.resolves([]); }); afterEach(() => { - globalState.folderHoldings = null; + globalState.folderStates.clear(); globalState.seedMarks.clear(); }); @@ -966,7 +1074,9 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); expect(result.holding, 'holdings went to a caller that proved nothing').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'] }); + expect(result).to.deep.equal({ + ready: true, folders: ['fluxw_w'], notLookedAt: [], heldOut: [], volumeGone: [], + }); }); }); @@ -983,7 +1093,9 @@ describe('appQueryService tests', () => { }); expect(result.holding, 'a body dated in the future served as a credential').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'] }); + expect(result).to.deep.equal({ + ready: true, folders: ['fluxw_w'], notLookedAt: [], heldOut: [], volumeGone: [], + }); }); // The bound is the fleet's clocks disagreeing, which is honest and must still work. @@ -1042,7 +1154,9 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body: signed({ target: '10.0.0.8:16127' }) }); expect(result.holding, 'a signature captured in flight worked on another node').to.equal(undefined); - expect(result).to.deep.equal({ ready: true, folders: ['fluxa_a'] }); + expect(result).to.deep.equal({ + ready: true, folders: ['fluxw_w'], notLookedAt: [], heldOut: [], volumeGone: [], + }); }); it('withholds the holdings when this node does not know its own address', async () => { @@ -1078,10 +1192,71 @@ describe('appQueryService tests', () => { const result = await appQueryService.promotedFolderHoldings({ body }); sinon.assert.calledOnceWithExactly(peerIdentityServiceStub.answerSealer, 'promoted', body); - expect(seal.firstCall.args[0]).to.have.keys('ready', 'folders', 'holding', 'seeding'); + expect(seal.firstCall.args[0]).to.have.keys('ready', 'folders', 'notLookedAt', 'heldOut', 'volumeGone', 'holding', 'seeding'); expect(result.sealed).to.equal(true); }); + // An install its monitor has not reached: a peer about to seed waits on it rather + // than read this node as holding nothing. + describe('the folders it has not yet looked at', () => { + beforeEach(() => { + fluxNetworkHelperStub.verifySignedFluxnodeMessage.resolves(true); + }); + + it('lists a folder not yet looked at, to any caller, and claims nothing of it', async () => { + globalState.folderStates.set('fluxb_b', { state: 'notLookedAt' }); + + const open = await appQueryService.promotedFolderHoldings({ body: {} }); + const signedAnswer = await appQueryService.promotedFolderHoldings({ body: signed() }); + + expect(open.notLookedAt).to.deep.equal(['fluxb_b']); + expect(signedAnswer.holding).to.not.have.property('fluxb_b'); + }); + + it('answers no claim for a folder it could not read and one it has sent', async () => { + globalState.folderStates.set('fluxn_n', { state: 'noClaim' }); + + const result = await appQueryService.promotedFolderHoldings({ body: signed() }); + + expect(result.notLookedAt).to.deep.equal([]); + expect(result.holding).to.deep.equal({ + fluxa_a: { bytes: 5821604997, newestModified: 200 }, + fluxn_n: { unreadable: true }, + }); + }); + + it('lists a folder whose volume will not mount by whether its image is gone, to any caller, and claims nothing of it', async () => { + globalState.folderStates.set('fluxh_h', { state: 'heldOut' }); + globalState.folderStates.set('fluxg_g', { state: 'volumeGone' }); + + const open = await appQueryService.promotedFolderHoldings({ body: {} }); + const signedAnswer = await appQueryService.promotedFolderHoldings({ body: signed() }); + + expect(open.heldOut).to.deep.equal(['fluxh_h']); + expect(open.volumeGone).to.deep.equal(['fluxg_g']); + expect(signedAnswer.holding).to.not.have.any.keys('fluxh_h', 'fluxg_g'); + }); + + it('leaves a writable folder out of both, whatever its state', async () => { + globalState.folderStates.set('fluxw_w', { state: 'notLookedAt' }); + + const result = await appQueryService.promotedFolderHoldings({ body: signed() }); + + expect(result.notLookedAt).to.deep.equal([]); + expect(result.holding).to.not.have.property('fluxw_w'); + }); + + // The route is open: the answer is what the monitor already recorded, read + // from memory, so a caller cannot make this node read its database. + it('reads no database to answer', async () => { + globalState.folderStates.set('fluxb_b', { state: 'notLookedAt' }); + + await appQueryService.promotedFolderHoldings({ body: {} }); + + sinon.assert.notCalled(dbHelperStub.findInDatabase); + }); + }); + it('answers an error when the answer cannot be signed, rather than send it unsigned', async () => { messageHelperStub.createErrorMessage.callsFake((message) => ({ status: 'error', data: { message } })); peerIdentityServiceStub.answerSealer.returns(() => Promise.reject(new Error('This node cannot sign as itself'))); diff --git a/tests/unit/appRebuild.test.js b/tests/unit/appRebuild.test.js new file mode 100644 index 0000000000..cb46a8b177 --- /dev/null +++ b/tests/unit/appRebuild.test.js @@ -0,0 +1,113 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); +const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutcome'); +const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); + +describe('appRebuild tests', () => { + let uninstaller; + let installer; + let appRebuild; + let bus; + const spec = { name: 'myapp', version: 8 }; + + beforeEach(() => { + uninstaller = { + removeAppLocally: sinon.stub().resolves(RemovalOutcome.REMOVED), + announceRemoval: sinon.stub().resolves(), + }; + installer = { + checkAppRequirements: sinon.stub().resolves(true), + registerAppLocally: sinon.stub().resolves(InstallOutcome.INSTALLED), + }; + bus = { publish: sinon.stub() }; + appRebuild = proxyquire('../../ZelBack/src/services/appLifecycle/appRebuild', { + '../utils/fluxEventBus': bus, + './appUninstaller': uninstaller, + './appInstaller': installer, + '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, + }); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('removes the app and installs it again, and tells the network nothing', async () => { + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.INSTALLED); + + sinon.assert.callOrder(uninstaller.removeAppLocally, installer.checkAppRequirements, installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('leaves the installer\'s teardown silent, so the network is told once, here', async () => { + await appRebuild.rebuildApp(spec); + + expect(installer.registerAppLocally.firstCall.args[4], 'the installer was asked to announce its own teardown').to.equal(false); + }); + + it('tells the network once when the app was removed and the install did not complete', async () => { + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + + sinon.assert.calledOnceWithExactly(uninstaller.announceRemoval, 'myapp'); + }); + + it('tells the network when the app was removed and its requirements are no longer met', async () => { + installer.checkAppRequirements.rejects(new Error('Insufficient RAM')); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + + sinon.assert.notCalled(installer.registerAppLocally); + sinon.assert.calledOnceWithExactly(uninstaller.announceRemoval, 'myapp'); + }); + + // Another removal held the node, so this one touched nothing: the caller can ask + // again, and the app is as it was. + it('answers busy, installing nothing and telling the network nothing, when another removal holds the node', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.BUSY); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.BUSY); + + sinon.assert.notCalled(installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('installs nothing and tells the network nothing when the removal did not finish', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + + sinon.assert.notCalled(installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('tells the network nothing about an app that was not installed here', async () => { + uninstaller.removeAppLocally.resolves(RemovalOutcome.NOT_INSTALLED); + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + + await appRebuild.rebuildApp(spec); + + sinon.assert.calledOnce(installer.registerAppLocally); + sinon.assert.notCalled(uninstaller.announceRemoval); + }); + + it('publishes how each rebuild ended', async () => { + await appRebuild.rebuildApp(spec); + uninstaller.removeAppLocally.resolves(RemovalOutcome.FAILED); + await appRebuild.rebuildApp(spec); + + expect(bus.publish.args).to.deep.equal([ + ['app:rebuilt', { name: 'myapp', outcome: InstallOutcome.INSTALLED }], + ['app:rebuilt', { name: 'myapp', outcome: InstallOutcome.FAILED }], + ]); + }); + + it('settles a removal it could not announce, and still answers', async () => { + installer.registerAppLocally.resolves(InstallOutcome.FAILED); + uninstaller.announceRemoval.rejects(new Error('no peers')); + + expect(await appRebuild.rebuildApp(spec)).to.equal(InstallOutcome.FAILED); + }); +}); diff --git a/tests/unit/appReconciler.test.js b/tests/unit/appReconciler.test.js index c2d7f6da39..0d0c32f3e6 100644 --- a/tests/unit/appReconciler.test.js +++ b/tests/unit/appReconciler.test.js @@ -5,6 +5,9 @@ const proxyquire = require('proxyquire').noCallThru(); // The real classifier: both the reconciler and the boot sweep classify a mount // fault through this one function, so the mock borrows it rather than restating it. const { classifyVolumeFault } = require('../../ZelBack/src/services/appTamperingDetectionService'); +// The real test of what counts as a database failure, borrowed for the same reason. +const { isDatabaseError } = require('../../ZelBack/src/services/dbHelper'); +const { MongoNetworkTimeoutError } = require('mongodb'); describe('appReconciler tests', () => { let appReconciler; @@ -21,6 +24,7 @@ describe('appReconciler tests', () => { dbHelper: { databaseConnection: () => ({ db: () => ({}) }), findOneInDatabase: sinon.stub().callsFake(async () => localSpec), + isDatabaseError, }, dockerService: { dockerContainerInspect: sinon.stub().resolves({ State: { Running: false, Status: 'exited', ExitCode: 1 } }), @@ -62,7 +66,6 @@ describe('appReconciler tests', () => { clearAppVolumeData: sinon.stub().resolves(), }, appsRuntimeState: { - isOperatorStopped: sinon.stub().resolves(false), operatorStopState: sinon.stub().resolves({ stopped: false, force: false }), operatorStoppedOrThrow: sinon.stub().resolves(false), restartWaitMs: sinon.stub().resolves(0), @@ -243,12 +246,8 @@ describe('appReconciler tests', () => { expect(stubs.dockerService.appDockerKill.called).to.be.false; }); - // The defect this closes: the lock and the mode were two reads of one - // document, and getState returns null for a read failure exactly as it does - // for "no record" - so a second read that failed reported no force flag and - // downgraded the operator's kill into a drain they never asked for. The mode - // now arrives with the decision that read it, so nothing later can contradict - // it. An empty getState here IS what that failed read looked like. + // The lock and the mode arrive together from the read that decided the stop, + // so no later read of the same document can contradict them. it('kills even when a later state read comes back empty', async () => { stubs.appsRuntimeState.operatorStopState.resolves({ stopped: true, force: true }); stubs.appsRuntimeState.getState.resolves(null); @@ -449,6 +448,45 @@ describe('appReconciler tests', () => { ).to.equal(false); }); + it('leaves a component alone while its app is departing, removal cleanup included', async () => { + stubs.globalState.departingApps.enter('App'); + + await appReconciler.reconcile('www_App'); + + expect(stubs.dockerService.dockerContainerInspect.called, 'a departing app was reconciled').to.be.false; + expect(stubs.dockerService.appDockerStart.called).to.be.false; + stubs.globalState.departingApps.leave('App'); + }); + + it('leaves a component alone while boot has not confirmed its app is still held here', async () => { + stubs.globalState.unconfirmedApps.set('App', Symbol('App')); + + await appReconciler.reconcile('www_App'); + + expect(stubs.dockerService.dockerContainerInspect.called, 'an unconfirmed app was reconciled').to.be.false; + expect(stubs.dockerService.appDockerStart.called).to.be.false; + stubs.globalState.unconfirmedApps.delete('App'); + }); + + it('starts nothing when the stop lock cannot be read, and fails the pass for a retry', async () => { + stubs.appsRuntimeState.operatorStopState.rejects(new Error('connection timed out')); + + const failure = await appReconciler.reconcile('www_App').catch((err) => err); + + expect(failure, 'the pass reported success on an unread lock').to.be.an('error'); + expect(stubs.dockerService.appDockerStart.called).to.be.false; + }); + + it('starts nothing when the restart generation cannot be read, and fails the pass for a retry', async () => { + stubs.appsRuntimeState.getState.rejects(new Error('connection timed out')); + + const failure = await appReconciler.reconcile('www_App').catch((err) => err); // inspect default: stopped + + expect(failure, 'the pass reported success on an unread generation').to.be.an('error'); + expect(stubs.dockerService.appDockerStart.called).to.be.false; + expect(stubs.appsRuntimeState.recordRestart.called, 'no attempt was made, so none is recorded').to.be.false; + }); + it('starts a stopped plain component that should run (default always policy)', async () => { await appReconciler.reconcile('www_App'); expect(stubs.appsRuntimeState.recordRestart.calledOnceWith('www_App')).to.be.true; @@ -754,6 +792,17 @@ describe('appReconciler tests', () => { expect(stubs.containerHealthMonitor.recreateMissingContainers.calledOnceWith('www_App')).to.be.true; }); + it('does not remove the app when the recreate fails on a database read, and retries', async () => { + stubs.dockerService.dockerContainerInspect.rejects(new Error('Container www_App not found')); + stubs.dockerService.dockerListContainers.resolves([]); // probe: docker is up + stubs.containerHealthMonitor.recreateMissingContainers.rejects(new MongoNetworkTimeoutError('connection timed out')); + + await appReconciler.reconcile('www_App'); + + expect(stubs.appUninstaller.removeAppLocally.called, 'an app was uninstalled because the database did not answer').to.be.false; + expect(stubs.appTamperingDetectionService.recordEvent.calledWithMatch('App', 'recreation_failed')).to.be.false; + }); + it('removes the app locally when recreation fails (docker reachable)', async () => { stubs.dockerService.dockerContainerInspect.rejects(new Error('Container www_App not found')); stubs.dockerService.dockerListContainers.resolves([]); // probe: docker is up @@ -822,6 +871,31 @@ describe('appReconciler tests', () => { expect(stubs.dockerService.appDockerStart.called).to.be.false; }); + // An update requests a restart of every component on every node that holds the + // app. On a standby that request must not start the g: component: the election + // decides where it runs. It stays pending, and a start the election later asks + // for satisfies it. + it('does not start a stopped g: component for a restart request while no controller has elected it', async () => { + localSpec = { name: 'App', version: 4, compose: [{ name: 'db', containerData: 'g:/data' }] }; + stubs.appsRuntimeState.getState.resolves({ restartGeneration: 2, actuatedRestartGeneration: 1 }); + await appReconciler.reconcile('db_App'); // controllerDesired unset, inspect default: stopped + expect(stubs.dockerService.appDockerStart.called, 'started').to.be.false; + expect(stubs.dockerService.appDockerRestart.called, 'restarted').to.be.false; + expect(stubs.appsRuntimeState.recordRestartGeneration.called, 'the request recorded as done').to.be.false; + }); + + it('does not start a stopped g: component for a restart request once a controller stood it down', async () => { + localSpec = { name: 'App', version: 4, compose: [{ name: 'db', containerData: 'g:/data' }] }; + stubs.globalState.bootContainerStateSettled = false; + appReconciler.setControllerDesired('db_App', 'stopped', 'masterSlave standby'); + stubs.globalState.bootContainerStateSettled = true; + stubs.appsRuntimeState.getState.resolves({ restartGeneration: 2, actuatedRestartGeneration: 1 }); + await appReconciler.reconcile('db_App'); // inspect default: stopped + expect(stubs.dockerService.appDockerStart.called, 'started').to.be.false; + expect(stubs.dockerService.appDockerRestart.called, 'restarted').to.be.false; + expect(stubs.appsRuntimeState.recordRestartGeneration.called, 'the request recorded as done').to.be.false; + }); + it('starts a g: component once a controller sets it running', async () => { localSpec = { name: 'App', version: 4, compose: [{ name: 'db', containerData: 'g:/data' }] }; // set desired without triggering the workqueue (boot gate closed -> enqueue held) @@ -2188,7 +2262,7 @@ describe('appReconciler tests', () => { }); describe('applyIntent serialises an intent write against a reconcile pass', () => { - // The defect this closes: a pass reads isOperatorStopped, then acts on that + // The defect this closes: a pass reads the stop lock, then acts on that // answer once docker has replied. An operator stop landing in that gap is // not seen, so the pass starts a container the operator has just stopped and // the next pass stops it again - the flap suite 45 catches. diff --git a/tests/unit/appSlice.test.js b/tests/unit/appSlice.test.js new file mode 100644 index 0000000000..e16338f59e --- /dev/null +++ b/tests/unit/appSlice.test.js @@ -0,0 +1,110 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const appSlice = require('../../ZelBack/src/services/appLifecycle/appSlice'); + +describe('appSlice tests', () => { + let root; + let runCommand; + // unit -> its cgroup, as systemd reports it + let units; + // systemd runs in a container here, so its tree starts below the filesystem's root + const SYSTEMD_ROOT = '/init'; + + function slice(unit, populated) { + const dir = path.join(root, SYSTEMD_ROOT, 'flux.slice', unit); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'cgroup.events'), `populated ${populated ? 1 : 0}\nfrozen 0\n`); + units.set('flux.slice', path.join(SYSTEMD_ROOT, 'flux.slice')); + units.set(unit, path.join(SYSTEMD_ROOT, 'flux.slice', unit)); + } + const stops = () => runCommand.getCalls().filter((call) => call.args[1].params[0] === 'stop'); + const stopped = () => stops().map((call) => call.args[1].params.at(-1)); + let stopResult; + + beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'appslice-')); + appSlice.useCgroupRoot(root); + units = new Map(); + stopResult = { error: null }; + runCommand = sinon.stub(serviceHelper, 'runCommand').callsFake((cmd, { params }) => Promise.resolve(params[0] === 'show' + ? { stdout: `${units.get(params[4]) ?? ''}\n`, error: null } + : stopResult)); + }); + + afterEach(() => { + appSlice.useCgroupRoot('/sys/fs/cgroup'); + fs.rmSync(root, { recursive: true, force: true }); + sinon.restore(); + }); + + describe('sliceName', () => { + it('names the app\'s slice under flux.slice', () => { + expect(appSlice.sliceName('palworld1790205120183')).to.equal('flux-palworld1790205120183.slice'); + }); + + it('escapes each hyphen, so one app\'s slice never nests inside another\'s', () => { + expect(appSlice.sliceName('my-app')).to.equal('flux-my\\x2dapp.slice'); + expect(appSlice.sliceName('my-app')).to.not.equal(`${appSlice.sliceName('my').slice(0, -'.slice'.length)}-app.slice`); + }); + + it('names no slice for a name no app can have', () => { + ['', 'a_b', 'a.b', '../x', 'a b', null, undefined].forEach((name) => { + expect(appSlice.sliceName(name), String(name)).to.equal(null); + }); + }); + }); + + describe('release', () => { + it('stops the app\'s slice once nothing lives in it', async () => { + slice('flux-my\\x2dapp.slice', false); + + await appSlice.release('my-app'); + + expect(stops().map((call) => call.args)).to.deep.equal([['systemctl', { runAsRoot: true, params: ['stop', '--no-block', 'flux-my\\x2dapp.slice'] }]]); + }); + + it('leaves a slice a process still lives in', async () => { + slice('flux-busy.slice', true); + + await appSlice.release('busy'); + + expect(stops()).to.be.empty; + }); + + it('does nothing for an app with no slice', async () => { + await appSlice.release('absent'); + + expect(stops()).to.be.empty; + }); + }); + + describe('stopEmptySlices', () => { + it('stops every empty app slice, and leaves those in use and anything else', async () => { + slice('flux-gone.slice', false); + slice('flux-gone\\x2dtoo.slice', false); + slice('flux-running.slice', true); + slice('other.slice', false); + + const count = await appSlice.stopEmptySlices(); + + expect(count).to.equal(2); + expect(stopped()).to.have.members(['flux-gone.slice', 'flux-gone\\x2dtoo.slice']); + }); + + it('counts no slice systemd could not stop', async () => { + slice('flux-gone.slice', false); + stopResult = { error: new Error('unit busy') }; + + expect(await appSlice.stopEmptySlices()).to.equal(0); + }); + + it('does nothing when there is no flux.slice', async () => { + expect(await appSlice.stopEmptySlices()).to.equal(0); + expect(stops()).to.be.empty; + }); + }); +}); diff --git a/tests/unit/appSpawner.test.js b/tests/unit/appSpawner.test.js index 0365fe4b5c..9fe0f3c72f 100644 --- a/tests/unit/appSpawner.test.js +++ b/tests/unit/appSpawner.test.js @@ -2,6 +2,8 @@ const { expect } = require('chai'); const { InstallOutcome } = require('../../ZelBack/src/services/utils/installOutcome'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); +const { MongoNetworkTimeoutError } = require('mongodb'); +const { isDatabaseError } = require('../../ZelBack/src/services/dbHelper'); const realPlacementFeasibility = require('../../ZelBack/src/services/appPlacement/placementFeasibility'); const realImageManager = require('../../ZelBack/src/services/appSecurity/imageManager'); const { resetGlobalState } = require('./fixtures/globalState'); @@ -125,7 +127,8 @@ describe('appSpawner tests', () => { '../dbHelper': { databaseConnection: sinon.stub().returns({ db: sinon.stub().returns({}) }), aggregateInDatabase: aggregateStub, - findInDatabase: sinon.stub().resolves([]), + findInDatabase: opts.localAppsRead ?? sinon.stub().resolves([]), + isDatabaseError, }, '../serviceHelper': { delay: delayStub, @@ -236,7 +239,7 @@ describe('appSpawner tests', () => { '../appNetwork/portManager': { ensureApplicationPortsNotUsed: sinon.stub().resolves(), siblingHoldingPort: sinon.stub().resolves(opts.siblingHoldingPort ?? null), - checkInstallingAppPortAvailable: sinon.stub().resolves({ + checkInstallingAppPortAvailable: (opts.portCheck ?? sinon.stub()).resolves({ ok: opts.portsAvailable ?? true, reason: (opts.portsAvailable ?? true) ? 'proven' : 'notOurs', }), @@ -1045,6 +1048,17 @@ describe('appSpawner tests', () => { sinon.assert.notCalled(installStub); }); + it('should test the ports for the application it is spawning, by name', async () => { + const portCheck = sinon.stub(); + buildModule({ + aggregateResult: [spawnableApp], appSpec: fullSpec, errorCount: 0, portsAvailable: false, portCheck, + }); + + await appSpawner.trySpawningGlobalApplication(); + + sinon.assert.calledOnceWithExactly(portCheck, sinon.match.array, fullSpec.name); + }); + it('should add to short-term cache when network error count >= 5', async () => { buildModule({ aggregateResult: [spawnableApp], appSpec: fullSpec, errorCount: 5 }); await appSpawner.trySpawningGlobalApplication().catch(() => {}); @@ -1052,6 +1066,38 @@ describe('appSpawner tests', () => { expect(globalStateStub.spawnErrorsLongerAppCache.has('abc123')).to.be.false; }); + it('tries the app again on the next pass when the installed apps cannot be read', async () => { + const installStub = sinon.stub().resolves(InstallOutcome.INSTALLED); + buildModule({ + aggregateResult: [spawnableApp], + appSpec: fullSpec, + errorCount: 0, + installStub, + localAppsRead: sinon.stub().rejects(new MongoNetworkTimeoutError('timed out')), + }); + + await appSpawner.trySpawningGlobalApplication().catch(() => {}); + + sinon.assert.notCalled(installStub); + expect(globalStateStub.trySpawningGlobalAppCache.has('abc123')).to.be.false; + }); + + it('holds the app back after an error that is not the database', async () => { + const installStub = sinon.stub().resolves(InstallOutcome.INSTALLED); + buildModule({ + aggregateResult: [spawnableApp], + appSpec: fullSpec, + errorCount: 0, + installStub, + localAppsRead: sinon.stub().rejects(new Error('bad record')), + }); + + await appSpawner.trySpawningGlobalApplication().catch(() => {}); + + sinon.assert.notCalled(installStub); + expect(globalStateStub.trySpawningGlobalAppCache.has('abc123')).to.be.true; + }); + it('should not block when network error count < 5', async () => { buildModule({ aggregateResult: [spawnableApp], appSpec: fullSpec, errorCount: 4 }); await appSpawner.trySpawningGlobalApplication().catch(() => {}); diff --git a/tests/unit/appSpecHelpers.test.js b/tests/unit/appSpecHelpers.test.js index 2a41564de8..0078d16ae5 100644 --- a/tests/unit/appSpecHelpers.test.js +++ b/tests/unit/appSpecHelpers.test.js @@ -868,6 +868,37 @@ describe('appSpecHelpers tests', () => { const response = await quote(buildSpec('g:/data', { priceUSD: 5.1 })); expect(response.data.usd).to.equal(5.1); }); + + [ + ['an app name the rule would reject', { name: 'Price_Test' }], + ['an owner the rule would reject', { owner: 'marketplace' }], + ].forEach(([description, change]) => { + // A non-enterprise spec never reaches the decrypt, so its name and owner + // are not checked here: it is priced as before. + it(`should price a non-enterprise spec with ${description}`, async () => { + const response = await quote(buildSpec('/data', change)); + expect(response.status).to.equal('success'); + }); + + it(`should refuse an enterprise spec with ${description} before decrypting it`, async () => { + const response = await quote(buildSpec('/data', { version: 8, enterprise: 'blob', ...change })); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + }); + + it('should price a marketplace spec: version 8, an empty enterprise field, a placeholder owner', async () => { + // The shape the marketplace install dialog sends. + const response = await quote(buildSpec('/data', { + version: 8, enterprise: '', owner: 'marketplace', expire: 22000, contacts: [], geolocation: [], nodes: [], staticip: false, + })); + expect(response.status).to.equal('success'); + }); + + it('should decrypt an enterprise spec with a valid app name and owner', async () => { + // Off ArcaneOS the decrypt refuses, which shows the spec reached it. + const response = await quote(buildSpec('/data', { version: 8, enterprise: 'blob' })); + expect(response.data.message).to.equal('Application Specifications can only be validated on a node running Arcane OS.'); + }); }); describe('module exports tests', () => { diff --git a/tests/unit/appStartupManager.test.js b/tests/unit/appStartupManager.test.js index e53acd5e5b..519c146711 100644 --- a/tests/unit/appStartupManager.test.js +++ b/tests/unit/appStartupManager.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai'); const sinon = require('sinon'); +const config = require('config'); const { resetGlobalState } = require('./fixtures/globalState'); const proxyquire = require('proxyquire').noCallThru(); @@ -16,9 +17,13 @@ describe('appStartupManager tests', () => { let appUninstallerStub; let globalStateStub; let appQueryServiceStub; + let appPlacementsStub; let portManagerStub; + let appSliceStub; + let nodeStatusMonitor; beforeEach(() => { + appSliceStub = { stopEmptySlices: sinon.stub().resolves(0) }; logStub = { info: sinon.stub(), warn: sinon.stub(), @@ -37,6 +42,8 @@ describe('appStartupManager tests', () => { fluxNetworkHelperStub = { getLocalSocketAddress: sinon.stub(), isNodeDos: sinon.stub().returns(false), + isStickyDosHeldBy: sinon.stub().returns(false), + StickyDosOwner: { HOST_STALL: 'hostStall' }, allowAppPorts: sinon.stub().resolves({ failed: [], locked: false }), }; @@ -59,7 +66,8 @@ describe('appStartupManager tests', () => { }; appUninstallerStub = { - removeAppLocally: sinon.stub().resolves(), + removeAppLocally: sinon.stub().resolves('removed'), + finishRemoval: sinon.stub(), }; const mockDb = { db: sinon.stub().returns('mockDatabase') }; @@ -75,6 +83,10 @@ describe('appStartupManager tests', () => { sinon.stub(globalStateStub, 'waitForDaemonReady').resolves(); sinon.stub(globalStateStub, 'waitForBootContainerStateSettled').resolves(); + appPlacementsStub = { + removeStrayPlacements: sinon.stub().resolves(0), + }; + appQueryServiceStub = { installedApps: sinon.stub().resolves({ status: 'success', data: [] }), decryptEnterpriseApps: sinon.stub().callsFake(async (apps) => ({ readable: apps, unreadable: [], inPlace: apps })), @@ -85,6 +97,10 @@ describe('appStartupManager tests', () => { '../../lib/log': logStub, }); + // The real module, fresh per test: its record of removals still running is what + // boot shares with the monitor. + nodeStatusMonitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', {}); + appStartupManager = proxyquire('../../ZelBack/src/services/appLifecycle/appStartupManager', { '../../lib/log': logStub, '../dbHelper': dbHelperStub, @@ -96,7 +112,23 @@ describe('appStartupManager tests', () => { './advancedWorkflows': advancedWorkflowsStub, '../appMonitoring/appReconciler': appReconcilerStub, './appUninstaller': appUninstallerStub, + './appSlice': appSliceStub, + '../appMonitoring/nodeStatusMonitor': nodeStatusMonitor, + '../appManagement/appPlacements': appPlacementsStub, '../utils/globalState': globalStateStub, + // runs the step again at once until it succeeds, as the real one does after a wait + '../utils/untilSucceeds': { + BOOT_STEP_RETRY_MS: 15000, + untilSucceeds: async (what, step) => { + for (;;) { + try { + return await step(); // eslint-disable-line no-await-in-loop + } catch { + // run it again + } + } + }, + }, '../appQuery/appQueryService': appQueryServiceStub, '../utils/appConstants': { localAppsInformation: 'localAppsInformation', SIGTERM_EXPIRY_MS: 420000, RUNNING_EXPIRY_MS: 7500000 }, '../utils/appUtilities': appUtilities, @@ -160,12 +192,12 @@ describe('appStartupManager tests', () => { expect(result).to.equal(true); }); - it('should return true on database error (fail-safe)', async () => { + it('answers unknown when the records cannot be read, neither valid nor expired', async () => { dbHelperStub.findInDatabase.rejects(new Error('DB connection lost')); const result = await appUtilities.appHasValidLocationOnNode('myApp', '10.0.0.1:16127'); - expect(result).to.equal(true); + expect(result).to.equal(null); }); it('should query with correct app name and IP', async () => { @@ -313,38 +345,192 @@ describe('appStartupManager tests', () => { expect(results.appsRemoved).to.deep.equal(['AppB']); }); - it('should record failure when removeAppLocally throws', async () => { + it('hands an app it could not remove to be finished, never starting it', async () => { dockerServiceStub.dockerListContainers.resolves([ { Names: ['/fluxAppA'], State: 'exited' }, ]); dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); - // Expired location dbHelperStub.findInDatabase.onSecondCall().resolves([]); - - appUninstallerStub.removeAppLocally.rejects(new Error('Remove failed')); + appUninstallerStub.removeAppLocally.resolves('failed'); const results = await appStartupManager.reconcileAppsOnBoot(); - expect(results.appsRemoved).to.deep.equal([]); - expect(results.appsFailed).to.have.lengthOf(1); - expect(results.appsFailed[0].app).to.equal('AppA'); - expect(results.appsFailed[0].error).to.equal('Remove failed'); + expect(results.appsFailed.map((failure) => failure.app)).to.deep.equal(['AppA']); + expect(appReconcilerStub.enqueue.calledWith('AppA'), 'an app boot could not remove was started').to.equal(false); + sinon.assert.calledOnceWithExactly(appUninstallerStub.finishRemoval, 'AppA', true, false); }); - it('should still start app when location DB check errors (fail-safe)', async () => { - dockerServiceStub.dockerListContainers.resolves([ - { Names: ['/fluxAppA'], State: 'exited' }, - ]); - dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); + describe('an app whose location record cannot be read', () => { + const futureExpiry = () => new Date(Date.now() + (300 * 1000)); + // the held app's check runs in the background, on stubs that answer at once + const settle = () => new Promise((resolve) => { setImmediate(resolve); }); - // Location check throws error - appHasValidLocationOnNode returns true (fail-safe) - dbHelperStub.findInDatabase.onSecondCall().rejects(new Error('DB error')); + beforeEach(() => { + dockerServiceStub.dockerListContainers.resolves([ + { Names: ['/fluxAppA'], State: 'exited' }, + ]); + dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'AppA' }]); + dbHelperStub.findInDatabase.onSecondCall().rejects(new Error('DB error')); + }); - const results = await appStartupManager.reconcileAppsOnBoot(); + it('is held, neither started nor removed', async () => { + // the record stays unreadable for as long as this test looks + dbHelperStub.findInDatabase.onThirdCall().returns(new Promise(() => {})); - expect(results.appsStarted).to.deep.equal(['AppA']); - expect(results.appsRemoved).to.deep.equal([]); + const results = await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + expect(results.appsUnconfirmed).to.deep.equal(['AppA']); + expect(globalStateStub.unconfirmedApps.has('AppA'), 'the app was not held').to.equal(true); + expect(appUninstallerStub.removeAppLocally.called, 'an app whose location is unknown was removed').to.equal(false); + expect(appUninstallerStub.finishRemoval.called, 'an app whose location is unknown was handed to be removed').to.equal(false); + expect(appReconcilerStub.enqueue.called, 'an app whose location is unknown was started').to.equal(false); + }); + + it('is started, and released, once its record reads valid', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([{ expireAt: futureExpiry() }]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appReconcilerStub.enqueue, 'AppA'); + expect(globalStateStub.unconfirmedApps.has('AppA'), 'a confirmed app is still held').to.equal(false); + expect(appUninstallerStub.removeAppLocally.called).to.equal(false); + }); + + it('has its record read again until it answers', async () => { + dbHelperStub.findInDatabase.onThirdCall().rejects(new Error('DB error')); + dbHelperStub.findInDatabase.onCall(3).rejects(new Error('DB error')); + dbHelperStub.findInDatabase.onCall(4).resolves([{ expireAt: futureExpiry() }]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + expect(dbHelperStub.findInDatabase.callCount, 'the record was not read until it answered').to.equal(5); + sinon.assert.calledOnceWithExactly(appReconcilerStub.enqueue, 'AppA'); + }); + + it('is removed once its record reads expired, telling the network nothing', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appUninstallerStub.removeAppLocally, 'AppA', null, true, true, false); + expect(appReconcilerStub.enqueue.called, 'an app whose location expired was started').to.equal(false); + expect(globalStateStub.unconfirmedApps.has('AppA')).to.equal(false); + }); + + it('stays held until its removal has run', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + let finishRemoval; + appUninstallerStub.removeAppLocally.returns(new Promise((resolve) => { finishRemoval = resolve; })); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + expect(globalStateStub.unconfirmedApps.has('AppA'), 'released while its removal was running').to.equal(true); + + finishRemoval('removed'); + await settle(); + expect(globalStateStub.unconfirmedApps.has('AppA')).to.equal(false); + }); + + it('is handed to be finished when its removal does not finish', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + appUninstallerStub.removeAppLocally.resolves('busy'); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appUninstallerStub.finishRemoval, 'AppA', true, false); + expect(appReconcilerStub.enqueue.called).to.equal(false); + }); + + describe('whose hold ends before its record is read', () => { + // The record answers only when the test says, after it has ended the hold. + const recordAnswersLater = () => { + let answer; + dbHelperStub.findInDatabase.onThirdCall().returns(new Promise((resolve) => { answer = resolve; })); + return (records) => answer(records); + }; + + it('is not removed when it was uninstalled meanwhile and its record reads expired', async () => { + const answer = recordAnswersLater(); + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + globalStateStub.unconfirmedApps.delete('AppA'); + answer([]); + await settle(); + + expect(appUninstallerStub.removeAppLocally.called, 'removed an app installed again after its uninstall').to.equal(false); + expect(appReconcilerStub.enqueue.called).to.equal(false); + }); + + it('is not started when it was uninstalled meanwhile and its record reads valid', async () => { + const answer = recordAnswersLater(); + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + globalStateStub.unconfirmedApps.delete('AppA'); + answer([{ expireAt: futureExpiry() }]); + await settle(); + + expect(appReconcilerStub.enqueue.called, 'started an app on the word of a hold that had ended').to.equal(false); + expect(appUninstallerStub.removeAppLocally.called).to.equal(false); + }); + + it('decides nothing for a later hold under the same name, and leaves that hold in place', async () => { + const answer = recordAnswersLater(); + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + const later = Symbol('AppA'); + globalStateStub.unconfirmedApps.set('AppA', later); + answer([]); + await settle(); + + expect(appUninstallerStub.removeAppLocally.called, 'decided an app another hold now covers').to.equal(false); + expect(globalStateStub.unconfirmedApps.get('AppA'), 'released a hold that was not its own').to.equal(later); + }); + + it('leaves a later hold in place when its own removal ends', async () => { + dbHelperStub.findInDatabase.onThirdCall().resolves([]); + const later = Symbol('AppA'); + appUninstallerStub.removeAppLocally.callsFake(async () => { + globalStateStub.unconfirmedApps.set('AppA', later); + return 'removed'; + }); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + expect(globalStateStub.unconfirmedApps.get('AppA'), 'released a hold set after its removal began').to.equal(later); + }); + }); + + it('has only the components boot starts started, when its g: components are left to the election', async () => { + dockerServiceStub.dockerListContainers.resolves([ + { Names: ['/fluxweb_MixedApp'], State: 'exited' }, + { Names: ['/fluxdb_MixedApp'], State: 'exited' }, + ]); + dbHelperStub.findInDatabase.onFirstCall().resolves([{ name: 'MixedApp' }]); + registryManagerStub.getApplicationGlobalSpecifications.withArgs('MixedApp').resolves({ + version: 8, + name: 'MixedApp', + compose: [ + { name: 'web', containerData: '' }, + { name: 'db', containerData: 'g:/data' }, + ], + }); + dbHelperStub.findInDatabase.onThirdCall().resolves([{ expireAt: futureExpiry() }]); + + await appStartupManager.reconcileAppsOnBoot(); + await settle(); + + sinon.assert.calledOnceWithExactly(appReconcilerStub.enqueue, 'web_MixedApp'); + }); }); it('should check location after g: syncthing check', async () => { @@ -502,6 +688,75 @@ describe('appStartupManager tests', () => { }); describe('manageAppsOnBoot', () => { + it('leaves a node out of service for a host stall to its own removal, and starts no app', async () => { + fluxNetworkHelperStub.isStickyDosHeldBy.returns(true); + fluxNetworkHelperStub.isNodeDos.returns(true); + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); + + await appStartupManager.manageAppsOnBoot({ machineRebooted: false, downtimeMs: 1000, cleanShutdown: true }); + + sinon.assert.calledWithExactly(fluxNetworkHelperStub.isStickyDosHeldBy, 'hostStall'); + sinon.assert.notCalled(appUninstallerStub.removeAppLocally); + sinon.assert.notCalled(globalStateStub.waitForDbReady); + }); + + it('moves past an app whose removal does not return within the deadline, and boot still settles', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'stuck' }, { name: 'next' }] }); + appUninstallerStub.removeAppLocally.withArgs('stuck').returns(new Promise(() => {})); + + const boot = appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); + await clock.tickAsync(config.fluxapps.appRemovalDeadlineMs - 1); + sinon.assert.neverCalledWith(appUninstallerStub.removeAppLocally, 'next'); + await clock.tickAsync(1); + await boot; + + sinon.assert.calledWith(appUninstallerStub.removeAppLocally, 'next'); + expect(globalStateStub.bootContainerStateSettled).to.equal(true); + } finally { + clock.restore(); + } + }); + + it('leaves a removal it stopped waiting for to run, and the monitor does not start it again', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + try { + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'stuck' }] }); + appUninstallerStub.removeAppLocally.withArgs('stuck').returns(new Promise(() => {})); + + const boot = appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); + await clock.tickAsync(config.fluxapps.appRemovalDeadlineMs); + await boot; + expect(globalStateStub.bootContainerStateSettled, 'fixture: boot moved past the removal').to.equal(true); + + const outcome = await nodeStatusMonitor.removeWithinDeadline(appUninstallerStub.removeAppLocally, 'stuck', true); + expect(outcome).to.equal(nodeStatusMonitor.STILL_REMOVING); + sinon.assert.calledOnce(appUninstallerStub.removeAppLocally); + } finally { + clock.restore(); + } + }); + + it('stops every empty app slice before it removes any app', async () => { + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); + + await appStartupManager.manageAppsOnBoot({ machineRebooted: true, downtimeMs: 8000000, cleanShutdown: false }); + + sinon.assert.calledOnce(appSliceStub.stopEmptySlices); + sinon.assert.callOrder(appSliceStub.stopEmptySlices, appUninstallerStub.removeAppLocally); + }); + + it('stops every empty app slice before it reconciles the apps', async () => { + dockerServiceStub.dockerListContainers.resolves([]); + dbHelperStub.findInDatabase.resolves([]); + + await appStartupManager.manageAppsOnBoot({ machineRebooted: false, downtimeMs: 1000, cleanShutdown: true }); + + sinon.assert.calledOnce(appSliceStub.stopEmptySlices); + sinon.assert.callOrder(appSliceStub.stopEmptySlices, globalStateStub.waitForDbReady); + }); + it('should reconcile on FluxOS-only restart (no stopped containers)', async () => { const bootContext = { machineRebooted: false, downtimeMs: 1000, cleanShutdown: true, @@ -515,6 +770,20 @@ describe('appStartupManager tests', () => { expect(appUninstallerStub.removeAppLocally.called).to.be.false; }); + it('removes nothing wholesale when the boot context could not be read', async () => { + const bootContext = { + unreadable: true, machineRebooted: true, downtimeMs: null, cleanShutdown: false, firstBoot: false, + }; + appQueryServiceStub.installedApps.resolves({ status: 'success', data: [{ name: 'app1' }] }); + dockerServiceStub.dockerListContainers.resolves([]); + dbHelperStub.findInDatabase.resolves([]); + + await appStartupManager.manageAppsOnBoot(bootContext); + + expect(appUninstallerStub.removeAppLocally.called, 'every app was removed on an unread downtime').to.be.false; + expect(globalStateStub.waitForDbReady.calledOnce, 'the apps are reconciled one by one instead').to.be.true; + }); + it('should remove all apps when clean shutdown and downtime > SIGTERM_EXPIRY', async () => { const bootContext = { machineRebooted: true, downtimeMs: 500000, cleanShutdown: true, @@ -675,7 +944,33 @@ describe('appStartupManager tests', () => { await appStartupManager.manageAppsOnBoot(KEEP); expect(logStub.error.calledWithMatch(/not opened in the firewall: helper crashed/)).to.equal(true); - expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(2); + // once for tidying the placement records, twice for the reconcile + expect(dbHelperStub.findInDatabase.callCount, 'the reconcile read the installed apps').to.equal(3); + }); + + it('removes stray records first, against the installed apps', async () => { + await appStartupManager.manageAppsOnBoot(KEEP); + + sinon.assert.calledOnceWithExactly(appPlacementsStub.removeStrayPlacements, ['web', 'db']); + sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, portManagerStub.appsWithPorts); + }); + + // A boot that stops before it reconciles the apps still clears stray records. + it('removes stray records on a boot that stops early', async () => { + fluxNetworkHelperStub.isNodeDos.returns(true); + + await appStartupManager.manageAppsOnBoot(KEEP); + + sinon.assert.calledOnce(appPlacementsStub.removeStrayPlacements); + }); + + it('removes stray records again until it succeeds, before it goes on', async () => { + appPlacementsStub.removeStrayPlacements.onFirstCall().rejects(new Error('mongo went away')); + + await appStartupManager.manageAppsOnBoot(KEEP); + + sinon.assert.calledTwice(appPlacementsStub.removeStrayPlacements); + sinon.assert.callOrder(appPlacementsStub.removeStrayPlacements, portManagerStub.appsWithPorts); }); it('opens nothing when the apps are all being removed', async () => { diff --git a/tests/unit/appSyncOrchestrator.test.js b/tests/unit/appSyncOrchestrator.test.js index 20fe7d796d..fd5e86ef0e 100644 --- a/tests/unit/appSyncOrchestrator.test.js +++ b/tests/unit/appSyncOrchestrator.test.js @@ -757,6 +757,26 @@ describe('AppSyncOrchestrator', () => { await orchestrator.stop(); }); + it('ends the request and asks another peer when its response cannot be recorded', async () => { + const peers = makeEligiblePeers(3); + const spare = makePeer('10.0.0.9:16127'); + getEligibleSyncPeersStub = sinon.stub().returns(peers); + + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peerEmitter.emit('peerThresholdReached', 12); + await clock.tickAsync(0); + + getEligibleSyncPeersStub.returns([peers[1], peers[2], spare]); + appSyncEvents.emit(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, peers[0].key); + await clock.tickAsync(0); + + expect(spare.send.called, 'a peer whose response was not recorded kept its slot').to.equal(true); + expect(orchestrator.isSyncResponseWanted(peers[0]), 'the rest of an unrecorded response was still being taken') + .to.equal(false); + await orchestrator.stop(); + }); + // The response path has the key and always had it. One arriving without a // peer means that path stopped saying, which is a fault to report. it('refuses to act on an unverifiable response that names no peer', async () => { @@ -796,6 +816,26 @@ describe('AppSyncOrchestrator', () => { return orchestrator; }; + it('does not credit a peer whose request ended while its last message was being stored', async () => { + const orchestrator = await driveToRequests(makeEligiblePeers(3)); + appSyncEvents.emit(EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, makeEligiblePeers(3)[0].key); + await clock.tickAsync(0); + + completeAllTypes(3, ['apprunning', 'appinstalling', 'apperrors', 'apptemp']); + await clock.tickAsync(0); + + expect(orchestrator.state, 'a peer was credited after its request had ended').to.equal(STATES.SYNCING); + }); + + it('a canary: the same completions with every request open complete the sync', async () => { + const orchestrator = await driveToRequests(makeEligiblePeers(3)); + + completeAllTypes(3, ['apprunning', 'appinstalling', 'apperrors', 'apptemp']); + await clock.tickAsync(0); + + expect(orchestrator.state).to.equal(STATES.READY); + }); + it('does not complete the sync on the three surveys alone', async () => { const orchestrator = await driveToRequests(makeEligiblePeers(3)); @@ -2063,10 +2103,81 @@ describe('AppSyncOrchestrator', () => { blockEmitter.emit('blocksProcessed', 2555000 + i); } await clock.tickAsync(0); + // the burst arrives inside one hash retry, which the build waits out + blockEmitter.emit('blocksProcessed', 2555261); + await clock.tickAsync(0); // The block timer fallback tries rebuildDb again expect(reindexStub.callCount).to.be.greaterThan(1); }); + + // A hash sync storing messages while the registry is unbuilt leaves their rows + // to the build, which would miss any stored after it read the log. + it('never runs the fallback\'s DB rebuild beside a hash sync', async () => { + reindexStub.onFirstCall().rejects(new Error('reindex failed')); + let finishRetry; + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peersUp(); + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + syncMissingHashesStub.callsFake(() => new Promise((resolve) => { finishRetry = resolve; })); + + for (let i = 1; i <= 260; i += 1) { + blockEmitter.emit('blocksProcessed', 2555000 + i); + } + await clock.tickAsync(0); + expect(finishRetry, 'no hash retry was under way, so nothing below was tested').to.be.a('function'); + expect(reindexStub.callCount, 'the DB rebuild ran beside a hash sync').to.equal(1); + + finishRetry({ resolved: 0, missing: 0, unreachable: 0, nextRetryHeight: null }); + await clock.tickAsync(0); + blockEmitter.emit('blocksProcessed', 2555261); + await clock.tickAsync(0); + expect(reindexStub.callCount, 'the DB rebuild was not run once the hash sync ended').to.equal(2); + }); + }); + + // An operator asks for a check of every missing app message. It is queued, and + // runs once nothing else holds the node. + describe('a requested hash check', () => { + it('runs at once when nothing holds the node', async () => { + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peersUp(); + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + syncMissingHashesStub.resetHistory(); + + appSyncEvents.emit(EVENTS.HASH_CHECK_REQUESTED); + await clock.tickAsync(0); + + sinon.assert.calledOnce(syncMissingHashesStub); + expect(syncMissingHashesStub.firstCall.args[0].force).to.equal(true); + }); + + it('waits for a hash sync under way, then runs on a later block', async () => { + let finishSync; + syncMissingHashesStub.callsFake(() => new Promise((resolve) => { finishSync = resolve; })); + const orchestrator = makeOrchestrator(); + orchestrator.start(defaultBootContext); + peersUp(); + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + expect(finishSync, 'no hash sync was under way, so nothing below was tested').to.be.a('function'); + + appSyncEvents.emit(EVENTS.HASH_CHECK_REQUESTED); + await clock.tickAsync(0); + expect(syncMissingHashesStub.getCalls().filter((c) => c.args[0].force), 'the check ran beside a hash sync').to.have.length(0); + + syncMissingHashesStub.resolves({ resolved: 0, missing: 0, unreachable: 0, nextRetryHeight: 9999999 }); + finishSync({ resolved: 0, missing: 0, unreachable: 0, nextRetryHeight: 9999999 }); + await clock.tickAsync(0); + blockEmitter.emit('blocksProcessed', 2555001); + await clock.tickAsync(0); + + expect(syncMissingHashesStub.getCalls().filter((c) => c.args[0].force), 'the queued check never ran').to.have.length(1); + }); }); describe('dbReady on fallback paths', () => { @@ -2153,7 +2264,9 @@ describe('AppSyncOrchestrator', () => { expect(syncMissingHashesStub.calledTwice).to.be.true; }); - it('should rebuild globalAppsInformation when a retry resolves messages, and only then', async () => { + // A retry's messages reach the registry as they are stored (appHashSyncService), so + // the registry is rebuilt in full only on the way to ready. + it('does not rebuild globalAppsInformation after a retry, whatever it resolved', async () => { syncMissingHashesStub.onFirstCall().resolves({ resolved: 0, missing: 2, unreachable: 0, nextRetryHeight: 2555200 }); syncMissingHashesStub.onSecondCall().resolves({ resolved: 0, missing: 2, unreachable: 0, nextRetryHeight: 2555300 }); syncMissingHashesStub.onThirdCall().resolves({ resolved: 2, missing: 0, unreachable: 0, nextRetryHeight: null }); @@ -2165,15 +2278,13 @@ describe('AppSyncOrchestrator', () => { await clock.tickAsync(0); const afterInitialSync = reindexStub.callCount; - // a retry that resolves nothing has nothing to apply blockEmitter.emit('blocksProcessed', 2555200); await clock.tickAsync(0); - expect(reindexStub.callCount).to.equal(afterInitialSync); - - // the messages a retry resolves are stored without promotion: they reach the specs by a rebuild blockEmitter.emit('blocksProcessed', 2555300); await clock.tickAsync(0); - expect(reindexStub.callCount).to.equal(afterInitialSync + 1); + + expect(syncMissingHashesStub.calledThrice, 'the retry that resolved messages never ran').to.equal(true); + expect(reindexStub.callCount).to.equal(afterInitialSync); }); it('should use fallback interval when no hashes are backed off', async () => { @@ -2274,6 +2385,27 @@ describe('AppSyncOrchestrator', () => { expect(versionCall.args[3]).to.deep.equal({ $set: { version: '8.12.0' } }); }); + it('writes no version marker while the reset fails, and runs the reset again on the next hash retry', async () => { + dbHelperStub.findOneInDatabase.resolves(null); + resetHashSyncForUpgradeStub.onFirstCall().rejects(new Error('db down')); + const versionWrites = () => findOneAndUpdateStub.getCalls().filter((c) => c.args[2]?._id === 'hashSyncVersion'); + + const orchestrator = makeOrchestrator({ fluxVersion: '8.12.0' }); + orchestrator.start(defaultBootContext); + + blockEmitter.emit('blocksProcessed', 2555000); + await clock.tickAsync(0); + + expect(syncMissingHashesStub.calledOnce).to.be.true; + expect(versionWrites()).to.have.length(0); + + blockEmitter.emit('blocksProcessed', 2555100); + await clock.tickAsync(0); + + expect(resetHashSyncForUpgradeStub.calledTwice).to.be.true; + expect(versionWrites()).to.have.length(1); + }); + it('should skip version check when fluxVersion not provided', async () => { const orchestrator = makeOrchestrator(); orchestrator.start(defaultBootContext); @@ -2372,15 +2504,15 @@ describe('AppSyncOrchestrator', () => { expect(ctx.downtimeMs).to.be.within(299000, 301000); }); - it('should return safe defaults on error', async () => { + it('reports an unread context as unreadable, neither a first boot nor a long absence', async () => { dbHelperStub.findOneInDatabase.rejects(new Error('DB down')); const ctx = await AppSyncOrchestrator.readBootContext(); - expect(ctx.machineRebooted).to.be.true; - expect(ctx.downtimeMs).to.equal(Infinity); + expect(ctx.unreadable).to.be.true; + expect(ctx.firstBoot).to.be.false; + expect(ctx.downtimeMs, 'an unknown downtime read as expired removes every app').to.equal(null); expect(ctx.cleanShutdown).to.be.false; - expect(ctx.firstBoot).to.be.true; }); }); @@ -2415,6 +2547,23 @@ describe('AppSyncOrchestrator', () => { await orchestrator.stop(); }); + it('clears the previous shutdown reason with each heartbeat until one lands', async () => { + const heartbeats = () => findOneAndUpdateStub.getCalls() + .filter((c) => c.args[2]?._id === 'heartbeat' && c.args[3]?.$set && 'lastAlive' in c.args[3].$set); + findOneAndUpdateStub.onFirstCall().rejects(new Error('connection timed out')); + const orchestrator = makeOrchestrator(); + await orchestrator.start(defaultBootContext); + // tests/unit/globalconfig: system.heartbeatIntervalMs + await clock.tickAsync(30000); + await clock.tickAsync(30000); + + const [failed, landed, after] = heartbeats(); + expect(failed.args[3].$unset, 'the first heartbeat carries the clear').to.deep.equal({ shutdownReason: '' }); + expect(landed.args[3].$unset, 'a failed clear is carried by the next heartbeat').to.deep.equal({ shutdownReason: '' }); + expect(after.args[3].$unset, 'cleared once it landed, never again').to.equal(undefined); + await orchestrator.stop(); + }); + it('should store boot context and expose via getter', async () => { const orchestrator = makeOrchestrator(); await orchestrator.start(defaultBootContext); diff --git a/tests/unit/appTamperingDetectionService.test.js b/tests/unit/appTamperingDetectionService.test.js index b60a828e1d..6d70035a51 100644 --- a/tests/unit/appTamperingDetectionService.test.js +++ b/tests/unit/appTamperingDetectionService.test.js @@ -446,21 +446,61 @@ describe('appTamperingDetectionService tests', () => { expect(logStub.debug.calledWithMatch(/repeated, incident count incremented/)).to.be.true; }); - it('no-ops when DB is not available', async () => { + it('writes an event once the database is available, under the time it was seen', async () => { + const clock = sinon.useFakeTimers({ now: new Date('2026-07-16T10:59:50Z'), toFake: ['setTimeout', 'Date'] }); + const connection = dbHelperStub.databaseConnection; dbHelperStub.databaseConnection = sinon.stub().returns(null); await service.recordEvent('myapp', 'container_vanished', 'x'); - expect(dbHelperStub.findOneAndUpdateInDatabase.called).to.be.false; + + dbHelperStub.databaseConnection = connection; + await clock.tickAsync(30 * 1000); + + expect(eventUpserts()).to.have.lengthOf(1); + const { query, update } = eventUpserts()[0]; + expect(query.incidentKey).to.equal(`unknown:${Math.floor(Date.parse('2026-07-16T10:59:50Z') / service.INCIDENT_BUCKET_MS)}`); + expect(update.$setOnInsert.firstSeen).to.deep.equal(new Date('2026-07-16T10:59:50Z')); + clock.restore(); }); - it('swallows write errors without throwing and logs them', async () => { + it('writes events that failed again, with a growing delay, until they are written', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + const write = dbHelperStub.findOneAndUpdateInDatabase; dbHelperStub.findOneAndUpdateInDatabase = sinon.stub().rejects(new Error('boom')); await service.recordEvent('myapp', 'container_vanished', 'x'); + await service.recordEvent('otherapp', 'mount_vanished', 'y'); - sinon.assert.calledOnce(logStub.error); + sinon.assert.calledTwice(logStub.error); expect(logStub.error.firstCall.args[0]).to.include('boom'); + + await clock.tickAsync(30 * 1000); + sinon.assert.calledThrice(dbHelperStub.findOneAndUpdateInDatabase); + await clock.tickAsync(30 * 1000); + sinon.assert.calledThrice(dbHelperStub.findOneAndUpdateInDatabase); + + dbHelperStub.findOneAndUpdateInDatabase = write; + await clock.tickAsync(30 * 1000); + + expect(eventUpserts().map((c) => c.query.appName)).to.deep.equal(['myapp', 'otherapp']); + clock.restore(); + }); + + it('does not write an incident without its owner when the installed apps cannot be read', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + installedApps.myapp = { owner: 'ownerZ', hash: 'hashH' }; + const read = dbHelperStub.findOneInDatabase; + dbHelperStub.findOneInDatabase = sinon.stub().rejects(new Error('timed out')); + + await service.recordEvent('myapp', 'container_vanished', 'x'); + expect(eventUpserts()).to.have.lengthOf(0); + + dbHelperStub.findOneInDatabase = read; + await clock.tickAsync(30 * 1000); + + expect(eventUpserts()[0].update.$setOnInsert.ownerZelid).to.equal('ownerZ'); + clock.restore(); }); }); @@ -755,13 +795,45 @@ describe('appTamperingDetectionService tests', () => { expect(dbHelperStub.removeDocumentsFromCollection.called).to.be.false; }); - it('swallows errors without throwing and logs them', async () => { + it('records the boot once the database answers, without throwing meanwhile', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); dbHelperStub.findOneInDatabase = sinon.stub().rejects(new Error('mongo down')); await service.checkNodeReboot(); sinon.assert.calledOnce(logStub.error); expect(logStub.error.firstCall.args[0]).to.include('mongo down'); + expect(historyUpdates()).to.have.lengthOf(0); + + setMarker({ _id: 'lastStartup', at: new Date(), bootId: PREVIOUS_BOOT_ID }); + await clock.tickAsync(30 * 1000); + + expect(historyUpdates()).to.have.lengthOf(1); + expect(markerUpdates()).to.have.lengthOf(1); + clock.restore(); + }); + + it('does not record a boot twice when the marker write after it failed', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'] }); + const history = { _id: 'bootHistory', boots: [] }; + dbHelperStub.findOneInDatabase = sinon.stub().callsFake(async (db, coll, query) => { + if (query._id === 'lastStartup') return { _id: 'lastStartup', bootId: PREVIOUS_BOOT_ID }; + if (query._id === 'bootHistory') return history; + return null; + }); + const write = dbHelperStub.findOneAndUpdateInDatabase; + dbHelperStub.findOneAndUpdateInDatabase = sinon.stub().callsFake(async (db, coll, query, update, options) => { + if (query._id === 'bootHistory') history.boots.push(...update.$push.boots.$each); + if (query._id === 'lastStartup' && dbHelperStub.findOneAndUpdateInDatabase.callCount === 2) throw new Error('mongo down'); + return write(db, coll, query, update, options); + }); + + await service.checkNodeReboot(); + await clock.tickAsync(30 * 1000); + + expect(history.boots.map((boot) => boot.bootId)).to.deep.equal([CURRENT_BOOT_ID]); + expect(markerUpdates()).to.have.lengthOf(1); + clock.restore(); }); }); @@ -787,16 +859,20 @@ describe('appTamperingDetectionService tests', () => { it('sums counts and keeps the widest window when merging duplicate rollups', async () => { findResults = [ { - appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 2, firstSeen: new Date(2000), lastSeen: new Date(5000), severity: 3, + _id: 'older', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 2, firstSeen: new Date(2000), lastSeen: new Date(5000), severity: 3, }, { - appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 5, firstSeen: new Date(1000), lastSeen: new Date(9000), severity: 3, + _id: 'newer', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 5, firstSeen: new Date(1000), lastSeen: new Date(9000), severity: 3, }, ]; await service.prepareIncidentRollup(); - expect(removed).to.deep.equal([{ appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1' }]); + expect(upserts.map(({ query }) => query)).to.deep.equal([{ _id: 'newer' }]); + expect(removed).to.deep.equal([{ + appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', _id: { $ne: 'newer' }, + }]); + sinon.assert.callOrder(dbHelperStub.updateOneInDatabase, dbHelperStub.removeDocumentsFromCollection); expect(upserts).to.have.lengthOf(1); const merged = upserts[0].set; expect(merged.count, 'counts were not summed').to.equal(7); @@ -830,6 +906,18 @@ describe('appTamperingDetectionService tests', () => { sinon.assert.calledOnce(createIndexStub); }); + it('removes no duplicate when the merged rollup cannot be written', async () => { + findResults = [ + { _id: 'a', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 2 }, + { _id: 'b', appName: 'www', eventType: 'mountReadonly', incidentKey: 'boot:1', count: 5 }, + ]; + dbHelperStub.updateOneInDatabase = sinon.stub().rejects(new Error('connection timed out')); + + await service.prepareIncidentRollup(); + + expect(removed).to.deep.equal([]); + }); + it('logs and does not throw when the merge fails, leaving it for the next boot', async () => { dbHelperStub.findInDatabase = sinon.stub().rejects(new Error('mongo mid-election')); diff --git a/tests/unit/appUninstaller.test.js b/tests/unit/appUninstaller.test.js index ed5bfc9deb..dbab26ad0e 100644 --- a/tests/unit/appUninstaller.test.js +++ b/tests/unit/appUninstaller.test.js @@ -4,8 +4,16 @@ const proxyquire = require('proxyquire').noCallThru(); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { RemovalOutcome } = require('../../ZelBack/src/services/utils/removalOutcome'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); describe('appUninstaller tests', () => { + // The placement record is its own module with its own tests; here it is the + // seam a removal writes through. Fresh per test. + let appPlacementsStub; + beforeEach(() => { + appPlacementsStub = { removePlacement: sinon.stub().resolves() }; + }); + describe('cleanupPorts tests', () => { let appUninstallerPorts; let firewallStub; @@ -18,6 +26,7 @@ describe('appUninstaller tests', () => { }; upnpStub = { isUPNP: sinon.stub().returns(true), removeMapUpnpPort: sinon.stub().resolves(true) }; appUninstallerPorts = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../fluxNetworkHelper': firewallStub, '../upnpService': upnpStub, '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, @@ -53,6 +62,7 @@ describe('appUninstaller tests', () => { let globalStateStub; let announceCycle; let getLocalSocketAddressStub; + let broadcastStub; let runCommandStub; beforeEach(() => { @@ -115,6 +125,7 @@ describe('appUninstaller tests', () => { runCommandStub = sinon.stub().resolves({ error: null, stdout: '', stderr: '' }); appUninstaller = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -426,6 +437,7 @@ describe('appUninstaller tests', () => { it('should handle app not found case', async () => { const appUninstallerWithDb = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -500,6 +512,7 @@ describe('appUninstaller tests', () => { it('should remove app locally if app name is specified and app in DB', async () => { const appUninstallerWithDbApp = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -594,9 +607,11 @@ describe('appUninstaller tests', () => { // after successful teardown only (failed removals clear nothing), on forced // and unforced paths alike. let runtimeStateStub; + let appSliceStub; - function buildUninstaller(spec, constantOverrides = {}) { + function buildUninstaller(spec, constantOverrides = {}, dbOverrides = {}) { runtimeStateStub = { remove: sinon.stub().resolves(), removeControllerState: sinon.stub().resolves() }; + appSliceStub = { release: sinon.stub().resolves() }; // The REAL departing tracker, taken fresh per test rather than // reimplemented here: a fake that counts differently from the module // would pass this suite over the defect the counting exists to prevent. @@ -604,21 +619,42 @@ describe('appUninstaller tests', () => { // is this suite's alone - evicting the entry instead would hand another // object to every module loaded after it, and globalState is a singleton // whose flags decide whether an operation may start at all. - const { departingApps, announceCycle: realAnnounceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {}); + const { departingApps, unconfirmedApps, announceCycle: realAnnounceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {}); announceCycle = realAnnounceCycle; getLocalSocketAddressStub = sinon.stub().resolves(null); + broadcastStub = sinon.stub().resolves(); globalStateStub = { departingApps, + unconfirmedApps, announceCycle, removalInProgress: false, installationInProgress: false, runningAppsCache: new Set(), receiveOnlySyncthingAppsCache: new Map(), - folderHoldings: new Map(), seedMarks: new Map(), + // the two operations this suite sets, named as the module names them + operationHolding() { + if (this.removalInProgress) return 'removal'; + if (this.installationInProgress) return 'installation'; + return null; + }, }; return proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, '../utils/globalState': globalStateStub, + // runs the step again after a moment until it succeeds, as the real one does after its wait + '../utils/untilSucceeds': { + BOOT_STEP_RETRY_MS: 15000, + untilSucceeds: async (what, step) => { + for (;;) { + try { + return await step(); // eslint-disable-line no-await-in-loop + } catch { + await new Promise((resolve) => { setTimeout(resolve, 10); }); // eslint-disable-line no-await-in-loop + } + } + }, + }, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -635,6 +671,7 @@ describe('appUninstaller tests', () => { findOneInDatabase: sinon.stub().resolves(spec), findInDatabase: sinon.stub().resolves([]), findOneAndDeleteInDatabase: sinon.stub().resolves(), + ...dbOverrides, }, '../dockerService': { appDockerKill: sinon.stub().resolves(), @@ -675,7 +712,7 @@ describe('appUninstaller tests', () => { '../fluxCommunicationMessagesSender': { broadcastMessageToOutgoing: sinon.stub().resolves(), broadcastMessageToIncoming: sinon.stub().resolves(), - broadcastMessageToAll: sinon.stub().resolves(), + broadcastMessageToAll: broadcastStub, }, '../appDatabase/registryManager': { availableApps: sinon.stub().resolves([]), @@ -690,6 +727,7 @@ describe('appUninstaller tests', () => { stopAppMonitoring: sinon.stub().resolves(), }, '../appManagement/appsRuntimeState': runtimeStateStub, + './appSlice': appSliceStub, 'node-cmd': { run: sinon.stub().callsFake((cmd, cb) => cb(null, '', '')) }, }); } @@ -721,6 +759,167 @@ describe('appUninstaller tests', () => { const res = null; // exercised without a response stream + describe('a cleanup step that fails after the containers are gone', () => { + // The app is part removed: it is held as departing - never started, never + // announced - and its removal runs again until it finishes, which tells the network. + const settle = async (done) => { + for (let i = 0; i < 200 && !done(); i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setTimeout(resolve, 10); }); + } + }; + + it('fails the removal, holds the app, and finishes the removal once the row can be deleted', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + let heldWhileFinishing = null; + deleteRow.onSecondCall().callsFake(async () => { + heldWhileFinishing = globalStateStub.departingApps.has('testapp'); + }); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true); + await settle(() => !globalStateStub.departingApps.has('testapp') && deleteRow.calledTwice); + + expect(outcome).to.equal('failed'); + expect(heldWhileFinishing, 'the part-removed app was not held while its removal was run again').to.be.true; + sinon.assert.calledTwice(deleteRow); + expect(globalStateStub.departingApps.has('testapp'), 'the hold outlived the removal').to.be.false; + sinon.assert.calledOnce(getLocalSocketAddressStub); + }); + + // A hard redeploy tells the network once its reinstall fails, and the cleanup + // that stopped part way is finished afterwards: that message is the one sent. + it('finishes a removal without telling the network again when it was told meanwhile', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + getLocalSocketAddressStub.resolves('10.0.0.1:16127'); + + // the reinstall holds the node, so the run that finishes the removal waits + globalStateStub.installationInProgress = true; + await uninstaller.removeAppLocally('testapp', res, true); + await uninstaller.announceRemoval('testapp'); + globalStateStub.installationInProgress = false; + await settle(() => deleteRow.calledTwice && !globalStateStub.departingApps.has('testapp')); + + sinon.assert.calledTwice(deleteRow); + sinon.assert.calledOnce(broadcastStub); + }); + + it('a canary: the same removal told nothing meanwhile tells the network once it finishes', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + getLocalSocketAddressStub.resolves('10.0.0.1:16127'); + + await uninstaller.removeAppLocally('testapp', res, true); + await settle(() => deleteRow.calledTwice && !globalStateStub.departingApps.has('testapp')); + + sinon.assert.calledOnce(broadcastStub); + }); + + it('holds an app the network was already told of, and does not tell it twice', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + let heldWhileFinishing = null; + deleteRow.onSecondCall().callsFake(async () => { + heldWhileFinishing = globalStateStub.departingApps.has('testapp'); + }); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + + await uninstaller.removeAppLocally('testapp', res, true, true, true); + await settle(() => !globalStateStub.departingApps.has('testapp') && deleteRow.calledTwice); + + expect(heldWhileFinishing, 'the part-removed app was not held while its removal was run again').to.be.true; + sinon.assert.calledOnce(getLocalSocketAddressStub); + }); + + it('removes the whole app when the removal of one of its components stops part way', async () => { + let heldWhileFinishing = null; + const deleteRow = sinon.stub().callsFake(async () => { + heldWhileFinishing = globalStateStub.departingApps.has('testapp'); + }); + const uninstaller = buildUninstaller(composedSpec, {}, { findOneAndDeleteInDatabase: deleteRow }); + runtimeStateStub.remove.onFirstCall().rejects(new Error('connection timed out')); + + const outcome = await uninstaller.removeAppLocally('comp1_testapp', res, true); + await settle(() => deleteRow.called && !globalStateStub.departingApps.has('testapp')); + + expect(outcome).to.equal('failed'); + sinon.assert.calledOnce(deleteRow); + expect(heldWhileFinishing, 'the app was not held while it was removed whole').to.be.true; + }); + + it('fails the removal, before the row, when the runtime state cannot be removed, and finishes it', async () => { + const deleteRow = sinon.stub().resolves(); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + runtimeStateStub.remove.onFirstCall().rejects(new Error('connection timed out')); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true); + expect(outcome).to.equal('failed'); + await settle(() => deleteRow.called); + + sinon.assert.calledOnce(deleteRow); + }); + + // A forced removal takes the node's removal lock without asking and clears it + // when it ends, so one run beside another removal would release that removal's + // lock part way through. The rerun waits for the node instead. + it('runs the removal again only once no other install or removal holds the node', async () => { + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + + globalStateStub.installationInProgress = true; + await uninstaller.removeAppLocally('testapp', res, true); + // time for the rerun to have run, had it not waited + await new Promise((resolve) => { setTimeout(resolve, 100); }); + sinon.assert.calledOnce(deleteRow); + expect(globalStateStub.installationInProgress, 'the rerun cleared another operation\'s hold').to.equal(true); + + globalStateStub.installationInProgress = false; + await settle(() => deleteRow.calledTwice && !globalStateStub.departingApps.has('testapp')); + sinon.assert.calledTwice(deleteRow); + }); + + it('runs one removal again for an app however often it is handed over', async () => { + let finishRow; + const deleteRow = sinon.stub().resolves(); + deleteRow.onFirstCall().rejects(new Error('connection timed out')); + deleteRow.onSecondCall().returns(new Promise((resolve) => { finishRow = resolve; })); + const uninstaller = buildUninstaller(v2Spec, {}, { findOneAndDeleteInDatabase: deleteRow }); + + await uninstaller.removeAppLocally('testapp', res, true); + await settle(() => deleteRow.calledTwice); + uninstaller.finishRemoval('testapp', true, true); + finishRow(); + await settle(() => !globalStateStub.departingApps.has('testapp')); + // time for a second run, had one been started, to reach the row + await new Promise((resolve) => { setTimeout(resolve, 100); }); + + sinon.assert.calledTwice(deleteRow); + }); + }); + + it('takes a forced removal\'s spec from the app\'s newest stored message, read by its name', async () => { + const messages = sinon.stub().resolves([{ height: 900, appSpecifications: v2Spec }]); + const uninstaller = buildUninstaller(null, {}, { findInDatabase: messages }); + const onRemoved = sinon.stub(); + uninstaller.setOnComponentRemoved(onRemoved); + + await uninstaller.removeAppLocally('testapp', res, true); + + sinon.assert.calledWithExactly( + messages, + sinon.match.any, + 'appsMessages', + { 'appSpecifications.name': 'testapp' }, + { projection: { _id: 0 }, sort: { height: -1, timestamp: -1 }, limit: 1 }, + ); + sinon.assert.calledOnceWithExactly(onRemoved, 'testapp'); + }); + it('notifies the seam with the bare name for a v1-3 app', async () => { const uninstaller = buildUninstaller(v2Spec); const onRemoved = sinon.stub(); @@ -743,6 +942,26 @@ describe('appUninstaller tests', () => { sinon.assert.calledWithExactly(onRemoved, 'comp2_testapp'); }); + it('releases the app\'s slice once the whole app is torn down', async () => { + const uninstaller = buildUninstaller(composedSpec); + + await uninstaller.removeAppLocally('testapp', res, true); + + sinon.assert.calledOnceWithExactly(appSliceStub.release, 'testapp'); + }); + + it('keeps the app\'s slice when one component is removed', async () => { + const uninstaller = buildUninstaller(composedSpec); + const onRemoved = sinon.stub(); + uninstaller.setOnComponentRemoved(onRemoved); + + await uninstaller.removeAppLocally('comp1_testapp', res, true); + + // the canary: the component was torn down + sinon.assert.calledOnceWithExactly(onRemoved, 'comp1_testapp'); + sinon.assert.notCalled(appSliceStub.release); + }); + it('scopes the seam to the one component on a component-scoped removal', async () => { const uninstaller = buildUninstaller(composedSpec); const onRemoved = sinon.stub(); @@ -760,21 +979,19 @@ describe('appUninstaller tests', () => { const uninstaller = buildUninstaller(composedSpec); // Keyed by the app IDENTIFIER, which is what the folder id and the cleanup // both use - not the component name the removal is asked for. - globalStateStub.folderHoldings = new Map([ - ['fluxcomp1_testapp', { bytes: 5821604997, newestModified: 200 }], - ['otherapp', { bytes: 4096, newestModified: 100 }], - ]); + folderStateMap.set('fluxcomp1_testapp', { state: 'measured', bytes: 5821604997, newestModified: 200 }); + folderStateMap.set('otherapp', { state: 'measured', bytes: 4096, newestModified: 100 }); globalStateStub.seedMarks.set('fluxcomp1_testapp', { stage: 'intent', bytes: 5821604997, newestModified: 200 }); globalStateStub.seedMarks.set('otherapp', { stage: 'intent', bytes: 4096, newestModified: 100 }); await uninstaller.removeAppLocally('comp1_testapp', res, true); - expect(globalStateStub.folderHoldings.has('fluxcomp1_testapp'), 'a removed app still claims to hold its data').to.equal(false); + expect(folderStateMap.has('fluxcomp1_testapp'), 'a removed app still claims to hold its data').to.equal(false); expect(globalStateStub.seedMarks.has('fluxcomp1_testapp'), 'a removed app still offers to seed').to.equal(false); expect(globalStateStub.seedMarks.has('otherapp')).to.equal(true); // Scoped to what was removed - a component-scoped removal is not a clear of // everything this node has answered for. - expect(globalStateStub.folderHoldings.has('otherapp')).to.equal(true); + expect(folderStateMap.has('otherapp')).to.equal(true); }); it('pairs the seam with the durable runtime-state clear (same identifiers)', async () => { @@ -812,6 +1029,55 @@ describe('appUninstaller tests', () => { sinon.assert.notCalled(runtimeStateStub.remove); }); + // The record follows the app's volume: a removal that deletes the volume + // deletes it, and one that keeps the volume keeps it. + describe('the placement record', () => { + it('goes with the app\'s volume', async () => { + const uninstaller = buildUninstaller(composedSpec); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, true); + + expect(outcome).to.equal(RemovalOutcome.REMOVED); + sinon.assert.calledOnceWithExactly(appPlacementsStub.removePlacement, 'testapp'); + }); + + it('a record that cannot be deleted leaves the removal standing', async () => { + const uninstaller = buildUninstaller(composedSpec); + appPlacementsStub.removePlacement.rejects(new Error('mongo went away')); + + const outcome = await uninstaller.removeAppLocally('testapp', res, true, false, true); + + expect(outcome).to.equal(RemovalOutcome.REMOVED); + expect(logStub.warn.calledWithMatch(/placement record of testapp was not removed: mongo went away/)).to.equal(true); + }); + + it('stays when one component is removed and the app with it is not', async () => { + const uninstaller = buildUninstaller(composedSpec); + + await uninstaller.removeAppLocally('comp1_testapp', res, true); + + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + + it('stays through a soft removal', async () => { + const uninstaller = buildUninstaller(composedSpec); + + await uninstaller.softRemoveAppLocally('testapp', null, { removalInProgress: false, installationInProgress: false }, sinon.stub()); + + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + + it('stays when the removal is refused', async () => { + const uninstaller = buildUninstaller(composedSpec); + globalStateStub.removalInProgress = true; + + const outcome = await uninstaller.removeAppLocally('testapp', res, false); + + expect(outcome).to.equal(RemovalOutcome.BUSY); + sinon.assert.notCalled(appPlacementsStub.removePlacement); + }); + }); + it('completes removal when no seam callback is registered', async () => { const uninstaller = buildUninstaller(v2Spec); @@ -821,6 +1087,46 @@ describe('appUninstaller tests', () => { sinon.assert.notCalled(logStub.error); }); + // Boot holds an app whose location record it could not read until a check + // decides it. An uninstall in the meantime leaves that check nothing to + // decide, and an install of the same name afterwards is another app. + describe('the boot hold', () => { + it('ends when the app is uninstalled', async () => { + const uninstaller = buildUninstaller(v2Spec); + globalStateStub.unconfirmedApps.set('testapp', Symbol('testapp')); + + await uninstaller.removeAppLocally('testapp', res, true, true, false); + + expect( + globalStateStub.unconfirmedApps.has('testapp'), + 'still held after its uninstall, so boot\'s check would decide an install that came after it', + ).to.be.false; + }); + + it('holds the app while its teardown runs', async () => { + const uninstaller = buildUninstaller(v2Spec); + globalStateStub.unconfirmedApps.set('testapp', Symbol('testapp')); + let heldDuring = null; + uninstaller.setOnComponentRemoved(() => { + heldDuring = globalStateStub.unconfirmedApps.has('testapp'); + }); + + await uninstaller.removeAppLocally('testapp', res, true, true, false); + + expect(heldDuring, 'released part way through its teardown, so it could be started or announced').to.be.true; + }); + + it('stays when a removal is refused', async () => { + const uninstaller = buildUninstaller(v2Spec); + globalStateStub.unconfirmedApps.set('testapp', Symbol('testapp')); + globalStateStub.removalInProgress = true; + + await uninstaller.removeAppLocally('testapp', res, false, true, false); + + expect(globalStateStub.unconfirmedApps.has('testapp'), 'released by a removal that never ran').to.be.true; + }); + }); + // A node stops claiming an app when it decides to hand it back, not when the // container happens to die. The app stays installed until the removal ends, so // without the mark the announcement built in that window re-creates the @@ -961,23 +1267,6 @@ describe('appUninstaller tests', () => { expect(getLocalSocketAddressStub.called, 'the removal never reached its broadcast').to.be.true; }); - // A cycle that never finishes must not hold the node's removals: they take the - // removal lock, and every install and redeploy queues behind that. - it('gives up on a wedged cycle and announces anyway', async () => { - const uninstaller = buildUninstaller(v2Spec, { ANNOUNCE_CYCLE_WAIT_MS: 50 }); - await announceCycle.enable(); - - const removal = uninstaller.removeAppLocally('testapp', res, true, true, true); - const raced = await Promise.race([ - removal.then(() => 'announced'), - new Promise((resolve) => { setTimeout(() => resolve('stuck'), 2000); }), - ]); - - expect(raced, 'a wedged announcement cycle held the removal indefinitely').to.equal('announced'); - expect(getLocalSocketAddressStub.called, 'gave up on the wait without going on to broadcast').to.be.true; - announceCycle.disable(); - }); - // A redeploy tells the network nothing, so it contradicts no announcement and // must not queue behind one - every spec change on the node would pay for it. it('does not wait for a cycle when the removal says nothing to the network', async () => { @@ -1036,6 +1325,7 @@ describe('appUninstaller tests', () => { db: sinon.stub().returns('appsDatabase'), }; const appUninstallerNoApp = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, @@ -1120,6 +1410,7 @@ describe('appUninstaller tests', () => { db: sinon.stub().returns('appsDatabase'), }; const appUninstallerWithApp = proxyquire('../../ZelBack/src/services/appLifecycle/appUninstaller', { + '../appManagement/appPlacements': appPlacementsStub, config: configStub, '../verificationHelper': verificationHelperStub, '../messageHelper': messageHelperStub, diff --git a/tests/unit/appValidator.test.js b/tests/unit/appValidator.test.js index 7d13880d8b..5594820ab7 100644 --- a/tests/unit/appValidator.test.js +++ b/tests/unit/appValidator.test.js @@ -1,6 +1,9 @@ +const { EventEmitter } = require('events'); const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); +const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/daemonServiceMiscRpcs'); +const appValidatorService = require('../../ZelBack/src/services/appRequirements/appValidator'); describe('appValidator tests', () => { let appValidator; @@ -1056,3 +1059,51 @@ describe('appValidator tests', () => { }); }); }); + +describe('appValidator spec verification route tests', () => { + const spec = { + version: 8, + name: 'verifytest', + description: 'verify test', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', + enterprise: 'blob', + }; + + const post = (handler, body) => new Promise((resolve) => { + const req = new EventEmitter(); + const res = { json: (response) => resolve(response) }; + handler(req, res); + req.emit('data', JSON.stringify(body)); + req.emit('end'); + }); + + beforeEach(() => { + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 2500000 } }); + }); + + afterEach(() => { + sinon.restore(); + }); + + [ + ['verifyAppRegistrationParameters', appValidatorService.verifyAppRegistrationParameters], + ['verifyAppUpdateApi', appValidatorService.verifyAppUpdateApi], + ].forEach(([route, handler]) => { + [ + ['an invalid app name', { name: 'verify_test' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`${route} should refuse an enterprise spec with ${description}`, async () => { + const response = await post(handler, { ...spec, ...change }); + expect(response.status).to.equal('error'); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + }); + + it(`${route} should decrypt a spec with a valid app name and owner`, async () => { + // Off ArcaneOS the decrypt refuses, which shows the spec reached it. + const response = await post(handler, spec); + expect(response.data.message).to.equal('Application Specifications can only be validated on a node running Arcane OS.'); + }); + }); +}); diff --git a/tests/unit/appsRuntimeState.test.js b/tests/unit/appsRuntimeState.test.js index 1b8aa737db..a911ef5f0f 100644 --- a/tests/unit/appsRuntimeState.test.js +++ b/tests/unit/appsRuntimeState.test.js @@ -7,10 +7,12 @@ describe('appsRuntimeState tests', () => { let store; // fake collection: identifier -> doc let logStub; let readFails; + let removeFails; beforeEach(() => { store = new Map(); readFails = false; + removeFails = false; logStub = { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }; const dbHelperStub = { @@ -63,7 +65,10 @@ describe('appsRuntimeState tests', () => { }); store.set(query.identifier, next); }, - removeDocumentsFromCollection: async (_db, _coll, query) => { store.delete(query.identifier); }, + removeDocumentsFromCollection: async (_db, _coll, query) => { + if (removeFails) throw new Error('connection timed out'); + store.delete(query.identifier); + }, }; appsRuntimeState = proxyquire('../../ZelBack/src/services/appManagement/appsRuntimeState', { @@ -112,11 +117,11 @@ describe('appsRuntimeState tests', () => { describe('operatorStopped', () => { it('persists the stop lock and reads it back', async () => { await appsRuntimeState.setOperatorStopped('www_App', true); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; }); it('defaults to not-stopped for an unknown component', async () => { - expect(await appsRuntimeState.isOperatorStopped('nope_App')).to.be.false; + expect(await appsRuntimeState.operatorStoppedOrThrow('nope_App')).to.be.false; }); it('clearing the lock (start) also clears the restart backoff history', async () => { @@ -125,7 +130,7 @@ describe('appsRuntimeState tests', () => { expect(store.get('www_App').restartHistory).to.have.lengthOf(2); await appsRuntimeState.setOperatorStopped('www_App', false); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.false; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.false; expect(store.get('www_App').restartHistory).to.deep.equal([]); }); @@ -237,7 +242,7 @@ describe('appsRuntimeState tests', () => { expect(await appsRuntimeState.requestOperatorStart('www_App')).to.be.true; - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; expect(await appsRuntimeState.operatorStopState('www_App')).to.deep.equal({ stopped: true, startRequested: true, force: false, asPrimary: true, }); @@ -272,7 +277,7 @@ describe('appsRuntimeState tests', () => { expect(await appsRuntimeState.releaseOperatorStart('www_App')).to.be.false; - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; }); it('leaves a lock the operator set again after asking for the start', async () => { @@ -282,7 +287,7 @@ describe('appsRuntimeState tests', () => { expect((await appsRuntimeState.operatorStopState('www_App')).startRequested).to.be.false; expect(await appsRuntimeState.releaseOperatorStart('www_App')).to.be.false; - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; }); it('does not report a start request once the lock is gone', async () => { @@ -328,7 +333,32 @@ describe('appsRuntimeState tests', () => { expect(requested, 'an unread lock was answered').to.be.an('error'); expect(released, 'an unread lock was answered').to.be.an('error'); readFails = false; - expect(await appsRuntimeState.isOperatorStopped('www_App'), 'the lock was lifted').to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App'), 'the lock was lifted').to.be.true; + }); + }); + + describe('a record that cannot be read', () => { + it('rejects rather than answering no record', async () => { + await appsRuntimeState.setOperatorStopped('www_App', true); + readFails = true; + + const state = await appsRuntimeState.getState('www_App').catch((err) => err); + const stop = await appsRuntimeState.operatorStopState('www_App').catch((err) => err); + + expect(state, 'an unread record was answered').to.be.an('error'); + expect(stop, 'an unread lock was answered').to.be.an('error'); + }); + + it('records no restart over a ladder it could not read', async () => { + await appsRuntimeState.recordRestart('www_App'); + await appsRuntimeState.recordRestart('www_App'); + const ladder = store.get('www_App').restartHistory; + readFails = true; + + await appsRuntimeState.recordRestart('www_App'); + + readFails = false; + expect(store.get('www_App').restartHistory).to.deep.equal(ladder); }); }); @@ -388,6 +418,16 @@ describe('appsRuntimeState tests', () => { await appsRuntimeState.remove('www_App'); expect(store.has('www_App')).to.be.false; }); + + it('remove rejects when the delete fails, rather than leaving the stop lock in silence', async () => { + await appsRuntimeState.setOperatorStopped('www_App', true); + removeFails = true; + + const failure = await appsRuntimeState.remove('www_App').catch((err) => err); + + expect(failure).to.be.an('error'); + expect(store.get('www_App').operatorStopped).to.be.true; + }); }); describe('backoff ladder reset requires a stable RUN (not time since the attempt)', () => { @@ -663,14 +703,14 @@ describe('appsRuntimeState tests', () => { // The module therefore normalizes at its own boundary. it('keys a docker-prefixed identifier and its bare form to the same document', async () => { await appsRuntimeState.setOperatorStopped('fluxwww_App', true); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.true; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.true; expect(store.has('fluxwww_App')).to.be.false; }); it('removes under either form', async () => { await appsRuntimeState.setOperatorStopped('www_App', true); await appsRuntimeState.remove('fluxwww_App'); - expect(await appsRuntimeState.isOperatorStopped('www_App')).to.be.false; + expect(await appsRuntimeState.operatorStoppedOrThrow('www_App')).to.be.false; }); }); @@ -961,6 +1001,49 @@ describe('appsRuntimeState tests', () => { // here survives only by being carried, from whichever twin holds it. The // field names below are deliberately not real ones - a real one would be // testing its own named rule instead. + it('writes the merge onto the newest twin before removing the others, and removes nothing when that write fails', async () => { + const ops = []; + let writeFails = false; + const orderState = proxyquire('../../ZelBack/src/services/appManagement/appsRuntimeState', { + '../../lib/log': logStub, + '../dbHelper': { + databaseConnection: () => ({ db: () => ({ collection: () => ({ createIndex: sinon.stub().resolves() }) }) }), + findInDatabase: async () => [ + { _id: 'older', identifier: 'www_App', operatorStopped: true, updatedAt: 1000 }, + { _id: 'newer', identifier: 'www_App', restartHistory: [5], updatedAt: 9000 }, + ], + removeDocumentsFromCollection: async (_db, _coll, query) => { ops.push(['remove', query]); }, + updateOneInDatabase: async (_db, _coll, query, update) => { + if (writeFails) throw new Error('connection timed out'); + ops.push(['write', query, update.$set]); + }, + }, + '../dockerService': { getBaseAppName: (id) => id }, + }); + + await orderState.prepareCollection(); + + expect(ops.map(([op]) => op)).to.deep.equal(['write', 'remove']); + expect(ops[0][1]).to.deep.equal({ _id: 'newer' }); + expect(ops[0][2].operatorStopped, 'the lock on the older twin').to.be.true; + expect(ops[1][1]).to.deep.equal({ identifier: 'www_App', _id: { $ne: 'newer' } }); + + ops.length = 0; + writeFails = true; + const error = await orderState.prepareCollection().catch((caught) => caught); + expect(error, 'boot is told the collection is not prepared').to.be.an('error'); + expect(ops, 'a twin was removed although the merge was never written').to.deep.equal([]); + }); + + it('rejects when the unique index cannot be built, so boot prepares the collection again', async () => { + docs = []; + createIndexStub.rejects(new Error('connection timed out')); + + const error = await prepState.prepareCollection().catch((caught) => caught); + + expect(error.message).to.equal('connection timed out'); + }); + it('carries fields the merge does not name, from whichever twin has them', async () => { docs = [ { diff --git a/tests/unit/availabilityChecker.test.js b/tests/unit/availabilityChecker.test.js index 255b111563..47795d4416 100644 --- a/tests/unit/availabilityChecker.test.js +++ b/tests/unit/availabilityChecker.test.js @@ -1,5 +1,6 @@ const { expect } = require('chai'); const sinon = require('sinon'); +const proxyquire = require('proxyquire'); // eslint-disable-next-line no-unused-vars const axios = require('axios'); const config = require('config'); @@ -12,6 +13,18 @@ const verificationHelper = require('../../ZelBack/src/services/verificationHelpe const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/daemonServiceMiscRpcs'); const upnpService = require('../../ZelBack/src/services/upnpService'); const networkStateService = require('../../ZelBack/src/services/networkStateService'); +const fluxHttpTestServer = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); +const portManager = require('../../ZelBack/src/services/appNetwork/portManager'); +const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); +const policyStore = require('../../ZelBack/src/services/policyStore'); +const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); + +const signer = { pubKey: 'pubkey', sign: () => 'signature' }; +// the check with a node that can sign its port test +const signingAvailabilityChecker = proxyquire('../../ZelBack/src/services/appMonitoring/availabilityChecker', { + '../utils/nodeSigner': { nodeSigner: async () => signer }, +}); describe('availabilityChecker tests', () => { let mockInstalledAppsFn; @@ -21,6 +34,8 @@ describe('availabilityChecker tests', () => { let isArcane; let delayStub; let setImmediateStub; + let udpServers; + let udpToken; beforeEach(() => { mockInstalledAppsFn = sinon.stub(); @@ -30,7 +45,7 @@ describe('availabilityChecker tests', () => { testingPort: null, nextTestingPort: null, originalPortFailed: null, - lastUPNPMapFailed: false, + routerRefusalsSeen: 0, }; mockPortsNotWorking = new Set(); mockFailedNodesCache = new Map(); @@ -40,10 +55,25 @@ describe('availabilityChecker tests', () => { delayStub = sinon.stub(serviceHelper, 'delay').resolves(); // Stub setImmediate to prevent infinite recursion setImmediateStub = sinon.stub(global, 'setImmediate'); + + // No application at this address holds a port, unless a test says so. + sinon.stub(registryManager, 'getRunningAppIpList').resolves([]); + sinon.stub(portManager, 'assignedPortsGlobalApps').resolves([]); + sinon.stub(portManager, 'siblingHoldingPort').resolves(null); + + udpServers = []; + udpToken = null; + sinon.stub(fluxUdpTestServer, 'FluxUdpTestServer').callsFake((token) => { + udpToken = token; + const server = { listen: sinon.stub().resolves(), allow: sinon.stub(), close: sinon.stub() }; + udpServers.push(server); + return server; + }); }); afterEach(() => { sinon.restore(); + fluxNetworkHelper.setUdpReachability(null); }); describe('checkMyAppsAvailability tests', () => { @@ -320,29 +350,57 @@ describe('availabilityChecker tests', () => { sinon.assert.calledWith(delayStub, 15_000); }); - it('should handle UPNP mapping failures', async () => { - const apps = []; + describe('a node behind UPnP', () => { + const UPNP_DOS = 'Not possible to run applications on the node, router returning exceptions when creating UPNP ports mappings'; + let refusals; + let mapUpnpPortStub; + let removeMapUpnpPortStub; + let portTestStub; - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ - data: { synced: true }, + const refused = (code, routerWide) => ({ + mapped: false, code, reason: `TCP 1 not mapped: UPnP error ${code}`, routerWide, + }); + const MAPPED = { + mapped: true, code: null, reason: null, routerWide: false, + }; + // the router refuses `count` mappings of others - an install's, a restore's + const routerRefusesOthers = (count) => { refusals += count; }; + + beforeEach(() => { + refusals = 0; + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); + sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); + sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); + mockInstalledAppsFn.resolves({ status: 'success', data: [] }); + sinon.stub(upnpService, 'isUPNP').returns(true); + sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); + // The port under test is drawn at random, so this gate has to be closed + // too: left real, it refuses whenever the draw lands in upnpBannedPorts + // and the run returns before it ever reaches mapUpnpPort. + sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); + sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); + sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); + sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); + sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); + sinon.stub(upnpService, 'routerRefusals').callsFake(() => refusals); + sinon.stub(upnpService, 'removalPending').returns(false); + mapUpnpPortStub = sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); + removeMapUpnpPortStub = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { + this.emit('listening'); + return this; + }); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'close').callsFake((callback) => callback && callback()); + portTestStub = sinon.stub(axios, 'post').resolves({ data: { status: 'success', data: { message: 'Ports reachable' } } }); }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: apps }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - // The port under test is drawn at random, so this gate has to be closed - // too: left real, it refuses whenever the draw lands in upnpBannedPorts - // and the run returns before it ever reaches mapUpnpPort. - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); // Failed - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); - await availabilityChecker.checkMyAppsAvailability( + // the test mapping refused as `result` gives it, one router-wide refusal counted when it is + const testMappingRefused = (result) => mapUpnpPortStub.callsFake(async () => { + if (result.routerWide) refusals += 1; + return result; + }); + + const pass = () => signingAvailabilityChecker.checkMyAppsAvailability( mockInstalledAppsFn, mockDosState, mockPortsNotWorking, @@ -350,58 +408,340 @@ describe('availabilityChecker tests', () => { isArcane, ); - expect(mockDosState.lastUPNPMapFailed).to.be.true; - }); + it('counts a test mapping the router refused for its own reason', async () => { + testMappingRefused(refused(728, true)); - it('opens the port under test to IPv4 only, and deletes only that rule', async () => { - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: [] }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - const allowTest = sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - const deleteTest = sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - const allowBoth = sinon.stub(fluxNetworkHelper, 'allowPort').resolves(); - const deleteBoth = sinon.stub(fluxNetworkHelper, 'deleteAllowPortRule').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); - - await availabilityChecker.checkMyAppsAvailability(mockInstalledAppsFn, mockDosState, mockPortsNotWorking, mockFailedNodesCache, isArcane); - - sinon.assert.calledOnceWithExactly(allowTest, mockDosState.testingPort); - sinon.assert.calledWith(deleteTest, mockDosState.testingPort); - sinon.assert.notCalled(allowBoth); - sinon.assert.notCalled(deleteBoth); - }); + await pass(); - it('should increase DOS state on repeated UPNP failures', async () => { - const apps = []; - mockDosState.lastUPNPMapFailed = true; // Already failed once + expect(mockDosState.portFailureScore).to.equal(4); + expect(mockDosState.dosStateValue, 'the node\'s DOS, which a count below the threshold leaves alone').to.equal(0); + sinon.assert.notCalled(portTestStub); + sinon.assert.calledWith(delayStub, 60_000); + }); - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ - data: { synced: true }, + it('does not count a test mapping refused for its port alone', async () => { + testMappingRefused(refused(718, false)); + + await pass(); + + expect(mockDosState.portFailureScore ?? 0).to.equal(0); }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: apps }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - // The port under test is drawn at random, so this gate has to be closed - // too: left real, it refuses whenever the draw lands in upnpBannedPorts - // and the run returns before it ever reaches mapUpnpPort. - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); - await availabilityChecker.checkMyAppsAvailability( + // A pass costs the router one mapping and one removal. + it('maps the port under test for one protocol a pass, TCP first', async () => { + await pass(); + + sinon.assert.calledOnceWithExactly(mapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App', { protocols: ['TCP'] }); + expect(JSON.parse(portTestStub.firstCall.args[1]).protocols).to.deep.equal(['tcp']); + sinon.assert.notCalled(fluxUdpTestServer.FluxUdpTestServer); + }); + + it('tests UDP alone on the pass after a TCP pass that passed, and TCP after that', async () => { + await pass(); + portTestStub.callsFake(async (url, body) => { + const { ports: [port] } = JSON.parse(body); + return { data: { status: 'success', data: { udpAnswered: { [port]: fluxUdpTestServer.udpReply(udpToken).toString() } } } }; + }); + await pass(); + + expect(mapUpnpPortStub.secondCall.args[2]).to.deep.equal({ protocols: ['UDP'] }); + expect(JSON.parse(portTestStub.secondCall.args[1]).protocols).to.deep.equal(['udp']); + expect(fluxHttpTestServer.FluxHttpTestServer.prototype.listen.callCount, 'a TCP test server on the UDP pass').to.equal(1); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + + await pass(); + expect(mapUpnpPortStub.thirdCall.args[2]).to.deep.equal({ protocols: ['TCP'] }); + }); + + // A peer that does not know `protocols` tests TCP, finds it unmapped, and + // says the port failed: on a UDP pass that is no evidence at all. + it('reads nothing from a peer\'s TCP failure on a UDP pass, and tests UDP again with another peer', async () => { + mockDosState.testingProtocol = 'udp'; + portTestStub.callsFake(async (url, body) => { + const { ports: [port] } = JSON.parse(body); + return { data: { status: 'error', data: { message: `Flux Applications on 1.2.3.4:16127 are not available. Failed port: ${port}` } } }; + }); + + await pass(); + + expect(mockPortsNotWorking.size).to.equal(0); + expect(mockDosState.portFailureScore ?? 0).to.equal(0); + expect(mockDosState.nextTestingPort).to.equal(mockDosState.testingPort); + expect(mockDosState.testingProtocol).to.equal('udp'); + sinon.assert.calledWith(delayStub.lastCall, 15_000); + }); + + it('goes back to TCP after as many UDP passes as learn nothing', async () => { + mockDosState.testingProtocol = 'udp'; + portTestStub.resolves({ data: { status: 'error', data: { message: 'Failed port: 1' } } }); + + await pass(); + await pass(); + expect(mockDosState.testingProtocol, 'gave up early').to.equal('udp'); + await pass(); + + expect(mockDosState.testingProtocol).to.equal('tcp'); + expect(mockDosState.udpUnsettled).to.equal(0); + }); + + // A router refusing the UDP mapping is as broken as one refusing TCP. + it('counts a UDP mapping the router refused for its own reason as a router fault, and stays on UDP', async () => { + mockDosState.testingProtocol = 'udp'; + testMappingRefused(refused(606, true)); + + await pass(); + + expect(mockDosState.portFailureScore).to.equal(4); + expect(mockDosState.testingProtocol).to.equal('udp'); + sinon.assert.notCalled(portTestStub); + }); + + it('stays on TCP while its TCP test fails', async () => { + portTestStub.resolves({ data: { status: 'error', data: { message: 'Failed port: 1' } } }); + + await pass(); + + expect(mockPortsNotWorking.size).to.equal(1); + expect(mockDosState.testingProtocol ?? 'tcp').to.equal('tcp'); + }); + + it('makes no test mapping, opens no port, and counts a failed check while the router has yet to remove a mapping of this node', async () => { + upnpService.removalPending.returns(true); + + await pass(); + + sinon.assert.notCalled(mapUpnpPortStub); + sinon.assert.notCalled(fluxNetworkHelper.allowTestPort); + sinon.assert.notCalled(portTestStub); + expect(mockDosState.portFailureScore).to.equal(4); + sinon.assert.calledWith(delayStub, 60_000); + }); + + it('holds the node out of service once the router has yet to remove a mapping for as many passes as the threshold takes', async () => { + upnpService.removalPending.returns(true); + + // eslint-disable-next-line no-restricted-syntax + for (let i = 0; i < 25; i += 1) { + // eslint-disable-next-line no-await-in-loop + await pass(); + } + + expect(mockDosState.portFailureScore).to.equal(100); + expect(mockDosState.dosMessage).to.equal(UPNP_DOS); + expect(mockDosState.dosStateValue).to.equal(100); + sinon.assert.calledWith(delayStub.lastCall, 300_000); + }); + + it('waits no longer than an error\'s wait after a pass that left a removal pending', async () => { + upnpService.removalPending.onFirstCall().returns(false).returns(true); + + await pass(); + + sinon.assert.calledOnce(portTestStub); + sinon.assert.calledWith(delayStub.lastCall, 60_000); + }); + + it('waits no longer than an error\'s wait after a passing test in a pass that counted a router refusal', async () => { + routerRefusesOthers(1); + + await pass(); + + sinon.assert.calledOnce(portTestStub); + sinon.assert.calledWith(delayStub.lastCall, 60_000); + }); + + it('waits no longer than an error\'s wait after a pass the router refused a mapping during', async () => { + portTestStub.callsFake(async () => { + routerRefusesOthers(1); + return { data: { status: 'success', data: { message: 'Ports reachable' } } }; + }); + + await pass(); + + expect(mockDosState.portFailureScore ?? 0, 'counted by the next pass').to.equal(0); + sinon.assert.calledWith(delayStub.lastCall, 60_000); + }); + + it('waits a DOS\'s wait after a pass that counted a router refusal while out of service', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + routerRefusesOthers(1); + + await pass(); + + sinon.assert.calledWith(delayStub.lastCall, 300_000); + }); + + it('waits the whole wait after a passing test that left nothing pending', async () => { + await pass(); + + sinon.assert.calledWith(delayStub.lastCall, 3_600_000); + }); + + // eslint-disable-next-line no-restricted-syntax + for (const event of ['mappingRefused', 'removalPending', 'removalsClear']) { + it(`starts its next pass at once on UPnP's ${event}`, async () => { + delayStub.returns(new Promise(() => {})); + let waited = false; + const done = pass().then(() => { waited = true; }); + // eslint-disable-next-line no-await-in-loop + for (let i = 0; i < 1000 && !delayStub.called; i += 1) await Promise.resolve(); + sinon.assert.calledOnceWithExactly(delayStub, 3_600_000); + expect(waited, 'still waiting').to.equal(false); + + upnpService.routerEvents.emit(event); + upnpService.routerEvents.emit(event); + await done; + + expect(waited).to.equal(true); + sinon.assert.calledOnce(setImmediateStub); + }); + } + + it('starts no pass on a wake while it is not waiting', async () => { + portTestStub.callsFake(async () => { + upnpService.routerEvents.emit('removalsClear'); + return { data: { status: 'success', data: { message: 'Ports reachable' } } }; + }); + + await pass(); + + sinon.assert.calledOnce(setImmediateStub); + sinon.assert.calledWith(delayStub.lastCall, 3_600_000); + }); + + it('removes what the router kept of a test mapping it refused', async () => { + testMappingRefused(refused(718, false)); + + await pass(); + + sinon.assert.calledOnceWithExactly(removeMapUpnpPortStub, mockDosState.testingPort, 'Flux_Test_App'); + }); + + it('opens the port under test to IPv4 only, and deletes only that rule', async () => { + testMappingRefused(refused(728, true)); + const allowBoth = sinon.stub(fluxNetworkHelper, 'allowPort').resolves(); + const deleteBoth = sinon.stub(fluxNetworkHelper, 'deleteAllowPortRule').resolves(); + + await pass(); + + expect(fluxNetworkHelper.allowTestPort.args).to.deep.equal([[mockDosState.testingPort, 'tcp']]); + expect(fluxNetworkHelper.deleteTestPortRule.args).to.deep.equal([[mockDosState.testingPort, 'tcp']]); + sinon.assert.notCalled(allowBoth); + sinon.assert.notCalled(deleteBoth); + }); + + it('counts once a pass however many mappings the router refused since the last', async () => { + routerRefusesOthers(7); + + await pass(); + + expect(mockDosState.portFailureScore).to.equal(4); + expect(mockDosState.routerRefusalsSeen).to.equal(7); + }); + + it('counts nothing for refusals an earlier pass counted', async () => { + routerRefusesOthers(7); + mockDosState.routerRefusalsSeen = 7; + + await pass(); + + expect(mockDosState.portFailureScore ?? 0).to.equal(0); + }); + + it('keeps its router DOS through a pass whose test worked while the router refused others', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + routerRefusesOthers(1); + + await pass(); + + sinon.assert.calledOnce(portTestStub); + expect(mockDosState.dosMessage).to.equal(UPNP_DOS); + expect(mockDosState.dosStateValue).to.equal(100); + expect(mockDosState.portFailureScore).to.equal(100); + }); + + it('ends its router DOS on a pass whose test worked and the router refused nothing', async () => { + mockDosState.portFailureScore = 100; + mockDosState.dosStateValue = 100; + mockDosState.dosMessage = UPNP_DOS; + routerRefusesOthers(3); + mockDosState.routerRefusalsSeen = 3; + + await pass(); + + sinon.assert.calledOnce(portTestStub); + expect(mockDosState.dosMessage).to.equal(null); + expect(mockDosState.dosStateValue).to.equal(0); + expect(mockDosState.portFailureScore).to.equal(0); + }); + + it('leaves another check\'s DOS message standing when its own UPnP failures reach the threshold', async () => { + mockDosState.portFailureScore = 96; + mockDosState.dosStateValue = 12; + mockDosState.dosMessage = 'Flux is not available for outside communication'; + testMappingRefused(refused(728, true)); + + await pass(); + + expect(mockDosState.portFailureScore).to.equal(100); + expect(mockDosState.dosStateValue).to.equal(12); + expect(mockDosState.dosMessage).to.equal('Flux is not available for outside communication'); + }); + + it('holds the node out of service with its own message when its UPnP failures reach the threshold and no other stands', async () => { + mockDosState.portFailureScore = 96; + testMappingRefused(refused(728, true)); + + await pass(); + + expect(mockDosState.dosMessage).to.equal(UPNP_DOS); + expect(mockDosState.dosStateValue).to.equal(100); + sinon.assert.calledWith(delayStub, 300_000); + }); + }); + + describe('proof over TCP and UDP', () => { + const PEERS = ['192.168.1.200:16127', '192.168.1.201:16127', '192.168.1.202:16127', '192.168.1.203:16127', '192.168.1.204:16127']; + let observer; + let portTestStub; + + // What a peer that reads the port and probes UDP answers: `tcp` and `udp` + // are 'ours', another application's bytes, or null for silence; `udp` + // undefined is a peer that does not probe UDP. + const reading = (tcp, udp) => async (url, body) => { + const { ports: [port] } = JSON.parse(body); + const value = (v, ours) => (v === 'ours' ? ours : v); + const data = { message: 'Ports reachable', answered: { [port]: value(tcp, `X-Flux-Port-Test: ${udpToken}`) } }; + if (udp !== undefined) data.udpAnswered = { [port]: value(udp, fluxUdpTestServer.udpReply(udpToken).toString()) }; + return { data: { status: 'success', data } }; + }; + + beforeEach(() => { + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); + sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); + sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); + mockInstalledAppsFn.resolves({ status: 'success', data: [] }); + sinon.stub(upnpService, 'isUPNP').returns(false); + sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); + observer = -1; + sinon.stub(networkStateService, 'getRandomExternalObserver').callsFake(async () => { + observer += 1; + return PEERS[observer]; + }); + sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(false); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'listen').callsFake(function listen() { + this.emit('listening'); + return this; + }); + sinon.stub(fluxHttpTestServer.FluxHttpTestServer.prototype, 'close').callsFake((callback) => callback && callback()); + portTestStub = sinon.stub(axios, 'post').callsFake(reading('ours', 'ours')); + }); + + const pass = () => signingAvailabilityChecker.checkMyAppsAvailability( mockInstalledAppsFn, mockDosState, mockPortsNotWorking, @@ -409,59 +749,159 @@ describe('availabilityChecker tests', () => { isArcane, ); - expect(mockDosState.portFailureScore).to.equal(4); - expect(mockDosState.dosStateValue, 'the node\'s DOS, which a count below the threshold leaves alone').to.equal(0); - }); + it('asks the peer to read the port and probe it over UDP, answering only that peer', async () => { + await pass(); - it('leaves another check\'s DOS message standing when its own UPnP failures reach the threshold', async () => { - mockDosState.lastUPNPMapFailed = true; - mockDosState.portFailureScore = 96; - mockDosState.dosStateValue = 12; - mockDosState.dosMessage = 'Flux is not available for outside communication'; + const body = JSON.parse(portTestStub.firstCall.args[1]); + expect(body.echo).to.equal(true); + expect(body.protocols).to.deep.equal(['tcp', 'udp']); + sinon.assert.calledOnceWithExactly(udpServers[0].listen, mockDosState.testingPort); + sinon.assert.calledOnceWithExactly(udpServers[0].allow, '192.168.1.200'); + expect(udpServers[0].allow.calledBefore(portTestStub)).to.equal(true); + sinon.assert.calledOnce(udpServers[0].close); + }); - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: [] }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); + it('passes, and reports UDP reachable, on our token over TCP and UDP', async () => { + await pass(); - await availabilityChecker.checkMyAppsAvailability(mockInstalledAppsFn, mockDosState, mockPortsNotWorking, mockFailedNodesCache, isArcane); + expect(mockPortsNotWorking.size).to.equal(0); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + sinon.assert.calledWith(delayStub.lastCall, 3_600_000); + }); - expect(mockDosState.portFailureScore).to.equal(100); - expect(mockDosState.dosStateValue).to.equal(12); - expect(mockDosState.dosMessage).to.equal('Flux is not available for outside communication'); - }); + // Whatever holds the port, this node cannot serve an application on it. + it('counts a port that answered as something other than this node as failed', async () => { + portTestStub.callsFake(reading('somebody else entirely', undefined)); - it('holds the node out of service with its own message when its UPnP failures reach the threshold and no other stands', async () => { - mockDosState.lastUPNPMapFailed = true; - mockDosState.portFailureScore = 96; + await pass(); - sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); - sinon.stub(generalService, 'isNodeStatusConfirmed').resolves(true); - sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('192.168.1.100:16127'); - mockInstalledAppsFn.resolves({ status: 'success', data: [] }); - sinon.stub(upnpService, 'isUPNP').returns(true); - sinon.stub(fluxNetworkHelper, 'isPortBanned').returns(false); - sinon.stub(fluxNetworkHelper, 'isPortUPNPBanned').returns(false); - sinon.stub(networkStateService, 'getRandomExternalObserver').resolves('192.168.1.200:16127'); - sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(true); - sinon.stub(fluxNetworkHelper, 'allowTestPort').resolves(); - sinon.stub(upnpService, 'mapUpnpPort').resolves(false); - sinon.stub(fluxNetworkHelper, 'deleteTestPortRule').resolves(); - sinon.stub(upnpService, 'removeMapUpnpPort').resolves(); + expect(mockPortsNotWorking.has(mockDosState.testingPort)).to.equal(true); + }); + + it('takes a peer that does not read ports at its word', async () => { + portTestStub.resolves({ data: { status: 'success', data: { message: 'Ports reachable' } } }); + + await pass(); + + expect(mockPortsNotWorking.size).to.equal(0); + expect(fluxNetworkHelper.getUdpReachability()).to.equal(null); + }); + + it('skips a port an application at this address holds, testing nothing', async () => { + sinon.stub(Math, 'random').returns(0.5); + const { fluxapps: { portMin, portMax } } = config; + const port = Math.floor(0.5 * (portMax - portMin) + portMin); + registryManager.getRunningAppIpList.resolves([{ name: 'neighbour' }]); + portManager.assignedPortsGlobalApps.resolves([{ name: 'neighbour', ports: [port] }]); + + await pass(); - await availabilityChecker.checkMyAppsAvailability(mockInstalledAppsFn, mockDosState, mockPortsNotWorking, mockFailedNodesCache, isArcane); + sinon.assert.calledWith(portManager.assignedPortsGlobalApps, ['neighbour']); + sinon.assert.notCalled(portTestStub); + expect(mockPortsNotWorking.size).to.equal(0); + }); + + it('skips a port a node sharing this address says it holds, testing nothing', async () => { + portManager.siblingHoldingPort.resolves({ address: '192.168.1.100:16137', port: 1 }); + + await pass(); + + sinon.assert.notCalled(portTestStub); + }); + + // UDP loses datagrams: one peer's silence is tested again by another. + it('counts nothing on one peer\'s silence over UDP, and tests the port again with another peer', async () => { + portTestStub.callsFake(reading('ours', null)); + + await pass(); + const port = mockDosState.testingPort; + + expect(mockPortsNotWorking.size).to.equal(0); + expect(mockDosState.nextTestingPort).to.equal(port); + sinon.assert.calledWith(delayStub.lastCall, 15_000); + expect(fluxNetworkHelper.getUdpReachability()).to.equal(null); + + await pass(); + + expect(mockDosState.testingPort).to.equal(port); + sinon.assert.calledWith(networkStateService.getRandomExternalObserver.secondCall, '192.168.1.100:16127', { exclude: [PEERS[0]] }); + }); - expect(mockDosState.dosMessage).to.match(/^Not possible to run applications on the node/); - expect(mockDosState.dosStateValue).to.equal(100); + // A pass whose TCP passes is a passing pass for TCP, whatever UDP needs next. + it('clears its failed ports and lifts its DOS on a TCP pass whose first peer heard nothing over UDP, and still retests UDP', async () => { + const count = sinon.spy(fluxEventBus, 'count'); + mockPortsNotWorking.add(1234); + mockDosState.portFailureScore = 8; + mockDosState.dosMessage = 'Ports tested not reachable from outside, DMZ or UPNP required! All ports that have failed: [1234]'; + mockDosState.dosStateValue = 100; + portTestStub.callsFake(reading('ours', null)); + + await pass(); + + expect(mockPortsNotWorking.size, 'failed ports left until the retest').to.equal(0); + expect(mockDosState.dosMessage, 'its DOS left until the retest').to.equal(null); + expect(mockDosState.nextTestingPort).to.equal(mockDosState.testingPort); + sinon.assert.calledWith(delayStub.lastCall, 15_000); + sinon.assert.calledWithExactly(count, 'availability:firstUdpSilence', 'passed', 'cleared'); + }); + + it('tallies the branch each pass took', async () => { + const count = sinon.spy(fluxEventBus, 'count'); + + await pass(); + + sinon.assert.calledWithExactly(count, 'availability:pass', 'passed', 'proven'); + sinon.assert.neverCalledWith(count, 'availability:firstUdpSilence'); + }); + + it('takes our token heard by a second peer over the first peer\'s silence', async () => { + portTestStub.onFirstCall().callsFake(reading('ours', null)); + + await pass(); + await pass(); + + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + expect(mockDosState.udpSuspect).to.equal(null); + }); + + it('reports UDP unreachable, and counts nothing, when two peers hear nothing and policy has not switched the DOS on', async () => { + portTestStub.callsFake(reading('ours', null)); + + await pass(); + await pass(); + + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ + reachable: false, port: mockDosState.testingPort, peers: ['192.168.1.200', '192.168.1.201'], + }); + expect(mockPortsNotWorking.size).to.equal(0); + }); + + it('counts the port failed when two peers hear nothing and policy has switched the DOS on', async () => { + sinon.stub(policyStore, 'getDocument').withArgs('udpportdos').returns(true); + portTestStub.callsFake(reading('ours', null)); + + await pass(); + await pass(); + + expect(mockPortsNotWorking.has(mockDosState.testingPort)).to.equal(true); + expect(fluxNetworkHelper.getUdpReachability().reachable).to.equal(false); + }); + + it('drops one peer\'s silence unconfirmed after as many passes as find no peer that probes UDP', async () => { + portTestStub.onFirstCall().callsFake(reading('ours', null)); + portTestStub.callsFake(reading('ours', undefined)); + + await pass(); + const port = mockDosState.testingPort; + await pass(); + await pass(); + expect(mockDosState.nextTestingPort, 'stopped retesting early').to.equal(port); + await pass(); + + expect(mockDosState.udpSuspect).to.equal(null); + expect(mockDosState.nextTestingPort).to.equal(null); + expect(fluxNetworkHelper.getUdpReachability()).to.equal(null); + }); }); it('should handle errors gracefully and retry', async () => { diff --git a/tests/unit/benchmarkService.test.js b/tests/unit/benchmarkService.test.js index 4e34cd7a63..55a57aac11 100644 --- a/tests/unit/benchmarkService.test.js +++ b/tests/unit/benchmarkService.test.js @@ -125,6 +125,69 @@ describe('benchmarkService tests', () => { }); }); + describe('app request tests', () => { + const appCalls = [ + ['getPublicKey', 'getpublickey'], + ['decryptRSAMessage', 'decryptrsamessage'], + ['encryptMessage', 'encryptmessage'], + ['decryptMessage', 'decryptmessage'], + ]; + const validRequest = { + fluxID: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', + appName: 'my-app1', + message: 'aGVsbG8=', + blockHeight: 2000000, + }; + const refused = { + status: 'error', + data: { code: undefined, name: undefined, message: 'Invalid app name or owner' }, + }; + let benchmarkStub; + + beforeEach(() => { + sinon.stub(fs, 'stat').resolves(true); + benchmarkStub = sinon.stub(fluxRpc.FluxRpc.prototype, 'run').resolves('called'); + }); + + afterEach(() => { + sinon.restore(); + }); + + appCalls.forEach(([fn, rpc]) => { + it(`${fn} should send a request with a valid app name and owner`, async () => { + const message = JSON.stringify(validRequest); + + const result = await benchmarkService[fn](message); + + expect(result).to.eql({ status: 'success', data: 'called' }); + sinon.assert.calledOnceWithExactly(benchmarkStub, rpc, { params: [message] }); + }); + + [ + ['an invalid app name', { appName: 'my_app' }], + ['an invalid owner', { fluxID: 'my-owner' }], + ].forEach(([description, change]) => { + it(`${fn} should refuse a request with ${description}`, async () => { + const message = JSON.stringify({ ...validRequest, ...change }); + + const result = await benchmarkService[fn](message); + + expect(result).to.eql(refused); + sinon.assert.notCalled(benchmarkStub); + }); + }); + + ['not json', 'null', '"myapp"', '[]'].forEach((message) => { + it(`${fn} should refuse ${message} as a request`, async () => { + const result = await benchmarkService[fn](message); + + expect(result).to.eql(refused); + sinon.assert.notCalled(benchmarkStub); + }); + }); + }); + }); + describe('getStatus tests', () => { let benchmarkStub; diff --git a/tests/unit/chainUtilities.test.js b/tests/unit/chainUtilities.test.js new file mode 100644 index 0000000000..75f9ef646b --- /dev/null +++ b/tests/unit/chainUtilities.test.js @@ -0,0 +1,38 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('chainUtilities tests', () => { + describe('getChainParamsPriceUpdates', () => { + const configStub = { + database: { chainparams: { database: 'chainparams', collections: { chainMessages: 'chainmessages' } } }, + fluxapps: { price: [{ height: 0, minPrice: 1 }, { height: 1000, minPrice: 2 }] }, + }; + + function load(findInDatabase) { + return proxyquire('../../ZelBack/src/services/utils/chainUtilities', { + config: configStub, + '../dbHelper': { databaseConnection: () => ({ db: () => ({}) }), findInDatabase }, + '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, + }); + } + + it('answers the configured forks and the chain\'s, oldest first', async () => { + const chainUtilities = load(sinon.stub().resolves([{ height: 500, message: 'p_1_2_3_4' }])); + + const forks = await chainUtilities.getChainParamsPriceUpdates(); + + expect(forks.map((fork) => fork.height)).to.deep.equal([0, 500, 1000]); + }); + + // A list without the chain's forks prices every app against stale prices, + // and one without the configured forks too prices nothing. + it('rejects when the chain\'s forks cannot be read', async () => { + const chainUtilities = load(sinon.stub().rejects(new Error('connection timed out'))); + + const outcome = await chainUtilities.getChainParamsPriceUpdates().catch((err) => err); + + expect(outcome, 'a price list was answered without the chain\'s forks').to.be.an('error'); + }); + }); +}); diff --git a/tests/unit/cpuBurstHelper.test.js b/tests/unit/cpuBurstHelper.test.js index 7c25f23b36..aa2fe4855f 100644 --- a/tests/unit/cpuBurstHelper.test.js +++ b/tests/unit/cpuBurstHelper.test.js @@ -12,13 +12,11 @@ describe('cpuBurstHelper tests', () => { let cpuBurstHelper; beforeEach(() => { - // Fresh require each time to reset the cache cpuBurstHelper = proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { './enterpriseConfig': { getEnterpriseAppOwners: () => ['16mzUh6byiQr7rnYQxKraDbeBPsEHYpSTW'], }, }); - cpuBurstHelper.resetBurstSupportCache(); }); afterEach(() => { @@ -153,51 +151,21 @@ describe('cpuBurstHelper tests', () => { }); }); - describe('isCpuBurstSupported', () => { - it('should return false when burst config is disabled', async () => { - const helper = proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { - config: { cpuBurst: { enabled: false, periodUs: 100000 } }, - }); - helper.resetBurstSupportCache(); - const result = await helper.isCpuBurstSupported(); - expect(result).to.be.false; + describe('isCpuBurstEnabled', () => { + const withBurst = (cpuBurst) => proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { + config: { cpuBurst }, }); - it('should return false when cgroups v2 is not present', async () => { - sinon.stub(fs.promises, 'access').rejects(new Error('ENOENT')); - cpuBurstHelper.resetBurstSupportCache(); - - const result = await cpuBurstHelper.isCpuBurstSupported(); - expect(result).to.be.false; + it('is on when the config switches it on', () => { + expect(withBurst({ enabled: true, periodUs: 100000 }).isCpuBurstEnabled()).to.equal(true); }); - it('should return false for kernel < 5.14', async () => { - sinon.stub(fs.promises, 'access').resolves(); - sinon.stub(fs.promises, 'readFile').resolves('Linux version 5.4.0-generic'); - cpuBurstHelper.resetBurstSupportCache(); - - const result = await cpuBurstHelper.isCpuBurstSupported(); - expect(result).to.be.false; + it('is off when the config switches it off', () => { + expect(withBurst({ enabled: false, periodUs: 100000 }).isCpuBurstEnabled()).to.equal(false); }); - it('should return true for kernel >= 5.14 with cgroups v2', async () => { - sinon.stub(fs.promises, 'access').resolves(); - sinon.stub(fs.promises, 'readFile').resolves('Linux version 6.1.0-generic'); - cpuBurstHelper.resetBurstSupportCache(); - - const result = await cpuBurstHelper.isCpuBurstSupported(); - expect(result).to.be.true; - }); - - it('should cache the result on subsequent calls', async () => { - sinon.stub(fs.promises, 'access').resolves(); - const readStub = sinon.stub(fs.promises, 'readFile').resolves('Linux version 6.1.0-generic'); - cpuBurstHelper.resetBurstSupportCache(); - - await cpuBurstHelper.isCpuBurstSupported(); - await cpuBurstHelper.isCpuBurstSupported(); - // readFile called once for kernel version (cached after that) - expect(readStub.callCount).to.equal(1); + it('is off when the config does not mention it', () => { + expect(withBurst(undefined).isCpuBurstEnabled()).to.equal(false); }); }); @@ -307,12 +275,14 @@ describe('cpuBurstHelper tests', () => { expect(result).to.be.false; }); - it('should return false when isCpuBurstSupported is false', async () => { - sinon.stub(fs.promises, 'access').rejects(new Error('ENOENT')); - cpuBurstHelper.resetBurstSupportCache(); + it('applies nothing while burst is switched off', async () => { + const writeStub = sinon.stub(fs.promises, 'writeFile').resolves(); + const helper = proxyquire('../../ZelBack/src/services/utils/cpuBurstHelper', { + config: { cpuBurst: { enabled: false, periodUs: 100000 } }, + }); - const result = await cpuBurstHelper.applyBurst(12345, 2.0); - expect(result).to.be.false; + expect(await helper.applyBurst(12345, 2.0)).to.equal(false); + expect(writeStub.called).to.equal(false); }); it('should write burstUs equal to quotaUs when supported and uncapped', async () => { @@ -320,10 +290,8 @@ describe('cpuBurstHelper tests', () => { sinon.stub(os, 'cpus').returns(new Array(32).fill({ model: 'stub', speed: 2400 })); const accessStub = sinon.stub(fs.promises, 'access').resolves(); const readStub = sinon.stub(fs.promises, 'readFile'); - readStub.withArgs('/proc/version').resolves('Linux version 6.1.0-generic'); readStub.withArgs('/proc/12345/cgroup').resolves('0::/system.slice/docker-x.scope\n'); const writeStub = sinon.stub(fs.promises, 'writeFile').resolves(); - cpuBurstHelper.resetBurstSupportCache(); const result = await cpuBurstHelper.applyBurst(12345, 2.0, 'test-app'); expect(result).to.be.true; diff --git a/tests/unit/cpuThrottleExemption.test.js b/tests/unit/cpuThrottleExemption.test.js new file mode 100644 index 0000000000..aa08cdc6f9 --- /dev/null +++ b/tests/unit/cpuThrottleExemption.test.js @@ -0,0 +1,135 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +const MODULE_PATH = '../../ZelBack/src/services/utils/cpuThrottleExemption'; + +const PREFIXES = ['enshrouded', 'palworld', 'rustserver', 'rustserveroxide', 'eurotrucksimulator2', '7daystodie']; + +// A fixed clock, so a timestamp "in the future" means the same thing on every run. +const NOW = Date.UTC(2026, 9, 17, 18, 0, 0); +const TS = String(Date.UTC(2026, 9, 16, 18, 56, 43, 381)); + +function load(document) { + const log = { error: sinon.stub(), info: sinon.stub(), warn: sinon.stub() }; + const policyStore = { getDocument: sinon.stub().returns(document) }; + return { module: proxyquire(MODULE_PATH, { '../policyStore': policyStore, '../../lib/log': log }), log, policyStore }; +} + +describe('cpuThrottleExemption', () => { + afterEach(() => sinon.restore()); + + describe('when the policy is unknown', () => { + it('exempts nothing, so the throttle keeps its current behaviour', () => { + const { module: m } = load(null); + expect(m.getExemptPrefixes()).to.equal(null); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + }); + }); + + describe('when the document is malformed', () => { + it('exempts nothing and says so, rather than trusting part of it', () => { + const { module: m, log } = load(['enshrouded', 'Palworld']); + expect(m.getExemptPrefixes()).to.equal(null); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + expect(log.error.called).to.equal(true); + }); + + it('logs a malformed document once per bundle, not once per app', () => { + const { module: m, log } = load(['enshrouded', 'Palworld']); + m.isCpuThrottleExempt(`enshrouded${TS}`, NOW); + m.isCpuThrottleExempt(`palworld${TS}`, NOW); + m.isCpuThrottleExempt(`valheim${TS}`, NOW); + expect(log.error.callCount).to.equal(1); + }); + + it('recovers when the store adopts a valid bundle after a malformed one', () => { + const { module: m, policyStore } = load(['Palworld']); + expect(m.isCpuThrottleExempt(`palworld${TS}`, NOW)).to.equal(false); + policyStore.getDocument.returns(['palworld']); + expect(m.isCpuThrottleExempt(`palworld${TS}`, NOW)).to.equal(true); + }); + + it('rejects a document that is not an array', () => { + const { module: m } = load({ enshrouded: true }); + expect(m.getExemptPrefixes()).to.equal(null); + }); + }); + + describe('when the bundle carries the document', () => { + it('exempts a hub game server: a listed prefix followed by its timestamp', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`palworld${TS}`, NOW)).to.equal(true); + }); + + it('compares case-insensitively, as app names are', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`PalWorld${TS}`, NOW)).to.equal(true); + }); + + it('reads a prefix that itself ends in a digit unambiguously', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`eurotrucksimulator2${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`7daystodie${TS}`, NOW)).to.equal(true); + }); + + it('matches the whole prefix, so one prefix never claims another product', () => { + const { module: m } = load(['rustserver']); + expect(m.isCpuThrottleExempt(`rustserver${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`rustserveroxide${TS}`, NOW)).to.equal(false); + }); + + it('does not exempt the prefix alone, or a name the hub did not mint', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt('enshrouded', NOW)).to.equal(false); + expect(m.isCpuThrottleExempt('enshroudedserver', NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(`enshrouded${TS}x`, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(`enshrouded_${TS}`, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(`myenshrouded${TS}`, NOW)).to.equal(false); + }); + + it('does not exempt a prefix the document does not list', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`wordpress${TS}`, NOW)).to.equal(false); + }); + + it('requires a plausible timestamp, not just thirteen digits', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt('enshrouded0000000000000', NOW), 'before the hub existed').to.equal(false); + expect(m.isCpuThrottleExempt('enshrouded9999999999999', NOW), 'centuries ahead').to.equal(false); + expect(m.isCpuThrottleExempt(`enshrouded${NOW + 2 * 24 * 60 * 60 * 1000}`, NOW), 'two days ahead').to.equal(false); + }); + + it('tolerates a browser clock running a little ahead of the node', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(`enshrouded${NOW + 60 * 60 * 1000}`, NOW)).to.equal(true); + }); + + it('answers false for anything that is not a name', () => { + const { module: m } = load(PREFIXES); + expect(m.isCpuThrottleExempt(undefined, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(null, NOW)).to.equal(false); + expect(m.isCpuThrottleExempt(1792177003381, NOW)).to.equal(false); + }); + + it('treats an empty list as known, with nothing exempt', () => { + const { module: m } = load([]); + expect(m.getExemptPrefixes()).to.deep.equal(new Set()); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + }); + + it('rebuilds the set only when the store adopts a new bundle', () => { + const document = [...PREFIXES]; + const { module: m, policyStore } = load(document); + const first = m.getExemptPrefixes(); + expect(m.getExemptPrefixes()).to.equal(first); + + policyStore.getDocument.returns(['valheim']); + const second = m.getExemptPrefixes(); + expect(second).to.not.equal(first); + expect(m.isCpuThrottleExempt(`valheim${TS}`, NOW)).to.equal(true); + expect(m.isCpuThrottleExempt(`enshrouded${TS}`, NOW)).to.equal(false); + }); + }); +}); diff --git a/tests/unit/cryptographicKeys.test.js b/tests/unit/cryptographicKeys.test.js index 17c04cb6f8..5b203f1137 100644 --- a/tests/unit/cryptographicKeys.test.js +++ b/tests/unit/cryptographicKeys.test.js @@ -64,7 +64,7 @@ describe('cryptographicKeys tests', () => { reloadModule(); try { - await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect.fail('Should have thrown error'); } catch (error) { expect(error.message).to.include('Arcane OS'); @@ -85,7 +85,7 @@ describe('cryptographicKeys tests', () => { }), }); - const result = await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + const result = await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect(result).to.equal(mockPublicKey); }); @@ -100,7 +100,7 @@ describe('cryptographicKeys tests', () => { }); try { - await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect.fail('Should have thrown error'); } catch (error) { expect(error.message).to.include('Error getting public key'); @@ -120,7 +120,7 @@ describe('cryptographicKeys tests', () => { }); try { - await cryptographicKeys.getAppPublicKey('1ABC123', 'TestApp', 1000); + await cryptographicKeys.getAppPublicKey('1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', 'TestApp', 1000); expect.fail('Should have thrown error'); } catch (error) { expect(error.message).to.include('Error getting public key to encrypt app enterprise content from SAS'); @@ -139,7 +139,7 @@ describe('cryptographicKeys tests', () => { }), }); - const fluxID = '1ABC123XYZ'; + const fluxID = '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uCXYZ'; const appName = 'MyTestApp'; const blockHeight = 123456; @@ -162,7 +162,7 @@ describe('cryptographicKeys tests', () => { }); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -195,13 +195,13 @@ describe('cryptographicKeys tests', () => { process.env.FLUXOS_PATH = '/some/path'; reloadModule(); sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); - sinon.stub(serviceHelper, 'ensureObject').returns({ owner: '1ABC123' }); // Missing name + sinon.stub(serviceHelper, 'ensureObject').returns({ owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' }); // Missing name sinon.stub(messageHelper, 'createErrorMessage').callsFake((msg) => ({ status: 'error', data: { message: msg }, })); - const appSpec = JSON.stringify({ owner: '1ABC123' }); + const appSpec = JSON.stringify({ owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' }); await callGetPublicKey(req, res, appSpec); @@ -215,7 +215,7 @@ describe('cryptographicKeys tests', () => { reloadModule(); sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -228,7 +228,7 @@ describe('cryptographicKeys tests', () => { })); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -245,7 +245,7 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -265,7 +265,7 @@ describe('cryptographicKeys tests', () => { })); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -276,6 +276,29 @@ describe('cryptographicKeys tests', () => { expect(res.json.firstCall.args[0].data).to.equal(mockPublicKey); }); + [ + ['an invalid app name', { owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'Test_App' }], + ['an invalid owner', { owner: 'my-owner', name: 'TestApp' }], + ].forEach(([description, request]) => { + it(`should refuse a request with ${description}`, async () => { + process.env.FLUXOS_PATH = '/some/path'; + reloadModule(); + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ + status: 'success', + data: { synced: true, height: 2000 }, + }); + const publicKeyCall = sinon.stub(benchmarkService, 'getPublicKey').resolves({ status: 'success', data: '{}' }); + + await callGetPublicKey(req, res, JSON.stringify(request)); + + sinon.assert.calledOnce(res.json); + expect(res.json.firstCall.args[0].status).to.equal('error'); + expect(res.json.firstCall.args[0].data.message).to.equal('Invalid app name or owner'); + sinon.assert.notCalled(publicKeyCall); + }); + }); + it('should handle request body in chunks', async () => { process.env.FLUXOS_PATH = '/some/path'; reloadModule(); @@ -283,9 +306,9 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); const ensureObjectStub = sinon.stub(serviceHelper, 'ensureObject'); - ensureObjectStub.onFirstCall().returns('{"owner":"1ABC123","name":"TestApp"}'); + ensureObjectStub.onFirstCall().returns('{"owner":"1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC","name":"TestApp"}'); ensureObjectStub.onSecondCall().returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -305,7 +328,7 @@ describe('cryptographicKeys tests', () => { })); // Send request body in multiple chunks - await callGetPublicKey(req, res, ['{"owner":"1ABC', '123","name":"T', 'estApp"}']); + await callGetPublicKey(req, res, ['{"owner":"1CbErtneaX2QV', 'yUfwU7JGB7VzvPgrgc3uC","name":"T', 'estApp"}']); sinon.assert.calledOnce(res.json); expect(res.json.firstCall.args[0].status).to.equal('success'); @@ -316,7 +339,7 @@ describe('cryptographicKeys tests', () => { reloadModule(); const verifyStub = sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -336,7 +359,7 @@ describe('cryptographicKeys tests', () => { }); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -352,7 +375,7 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -372,7 +395,7 @@ describe('cryptographicKeys tests', () => { }); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); @@ -389,7 +412,7 @@ describe('cryptographicKeys tests', () => { sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); sinon.stub(serviceHelper, 'ensureObject').returns({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ @@ -406,7 +429,7 @@ describe('cryptographicKeys tests', () => { })); const appSpec = JSON.stringify({ - owner: '1ABC123', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', name: 'TestApp', }); diff --git a/tests/unit/daemonHealthMonitor.test.js b/tests/unit/daemonHealthMonitor.test.js index b291ac7ac8..65c63e840d 100644 --- a/tests/unit/daemonHealthMonitor.test.js +++ b/tests/unit/daemonHealthMonitor.test.js @@ -300,6 +300,23 @@ describe('daemonHealthMonitor tests', () => { expect(logStub.error.calledWith('Error in daemon health check: RPC error')).to.be.true; }); + it('should try the removal again when the installed apps could not be read', async () => { + clock = sinon.useFakeTimers(); + + registryManagerStub.getInstalledApps.rejects(new Error('connection timed out')); + + daemonServiceMiscRpcsStub.isDaemonSynced.returns({ data: { synced: false } }); + await daemonHealthMonitor.checkDaemonHealthAndCleanup(); + clock.tick(2 * 60 * 60 * 1000); + await daemonHealthMonitor.checkDaemonHealthAndCleanup(); + expect(appUninstallerStub.removeAppLocally.called).to.be.false; + + registryManagerStub.getInstalledApps.resolves([{ name: 'app1' }]); + await daemonHealthMonitor.checkDaemonHealthAndCleanup(); + + expect(appUninstallerStub.removeAppLocally.callCount).to.equal(1); + }); + it('should reset allAppsRemoved flag when daemon recovers', async () => { clock = sinon.useFakeTimers(); diff --git a/tests/unit/databaseHealthService.test.js b/tests/unit/databaseHealthService.test.js new file mode 100644 index 0000000000..7910c542b8 --- /dev/null +++ b/tests/unit/databaseHealthService.test.js @@ -0,0 +1,164 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const config = require('config'); +const { EventEmitter } = require('events'); +const { MongoClient } = require('mongodb'); +const log = require('../../ZelBack/src/lib/log'); +const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); +const databaseHealthService = require('../../ZelBack/src/services/databaseHealthService'); +const { requireMongo } = require('./dbTestHelper'); + +const { suspectWindowMs } = config.database; +const MINUTE = 60 * 1000; + +describe('databaseHealthService tests', () => { + let clock; + let client; + + const created = (id) => client.emit('connectionCreated', { connectionId: id }); + const closed = (id, reason = 'error') => client.emit('connectionClosed', { connectionId: id, reason }); + const checkOutFailed = (reason = 'timeout') => client.emit('connectionCheckOutFailed', { reason }); + const timeOut = (n) => { for (let i = 0; i < n; i += 1) closed(100 + i); }; + const held = () => fluxNetworkHelper.isPlacementHeld(); + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout', 'hrtime'] }); + client = new EventEmitter(); + databaseHealthService.watch(client, 'main'); + }); + + afterEach(() => { + databaseHealthService.stop(); + clock.restore(); + sinon.restore(); + }); + + it('holds the node back from new placements on the fourth round trip that runs out', () => { + timeOut(3); + expect(held()).to.equal(false); + + timeOut(1); + + expect(held()).to.equal(true); + expect(databaseHealthService.isSuspect()).to.equal(true); + expect(fluxNetworkHelper.getPlacementHold()).to.include('Database calls are timing out'); + }); + + it('counts a wait for a pooled connection that ran out, and no other checkout failure', () => { + checkOutFailed('poolClosed'); + checkOutFailed('connectionError'); + timeOut(3); + expect(held()).to.equal(false); + + checkOutFailed('timeout'); + + expect(held()).to.equal(true); + }); + + it('does not count a connection closed for any reason but an error', () => { + ['stale', 'idle', 'poolClosed'].forEach((reason) => closed(1, reason)); + timeOut(3); + expect(held()).to.equal(false); + + closed(200); + + expect(held()).to.equal(true); + }); + + it('does not count the connections a cleared pool closes, and counts those created after it', () => { + [1, 2, 3, 4, 5].forEach(created); + client.emit('connectionPoolCleared', {}); + [1, 2, 3, 4, 5].forEach((id) => closed(id)); + expect(held()).to.equal(false); + + [6, 7, 8, 9].forEach(created); + [6, 7, 8].forEach((id) => closed(id)); + expect(held()).to.equal(false); + closed(9); + + expect(held()).to.equal(true); + }); + + it('counts only the round trips inside the window', () => { + timeOut(3); + clock.tick(suspectWindowMs); + + timeOut(1); + + expect(held()).to.equal(false); + }); + + it('releases the hold a window after the last round trip that ran out, and each one moves it', () => { + timeOut(4); + clock.tick(suspectWindowMs - MINUTE); + timeOut(1); + clock.tick(suspectWindowMs - MINUTE); + expect(held()).to.equal(true); + + clock.tick(MINUTE); + + expect(held()).to.equal(false); + expect(databaseHealthService.isSuspect()).to.equal(false); + }); + + it('counts afresh after a release', () => { + timeOut(4); + clock.tick(suspectWindowMs); + expect(held()).to.equal(false); + + timeOut(3); + + expect(held()).to.equal(false); + }); + + it('counts the round trips of every client it watches together', () => { + const maintenance = new EventEmitter(); + databaseHealthService.watch(maintenance, 'maintenance'); + timeOut(2); + maintenance.emit('connectionClosed', { connectionId: 1, reason: 'error' }); + expect(held()).to.equal(false); + + maintenance.emit('connectionClosed', { connectionId: 2, reason: 'error' }); + + expect(held()).to.equal(true); + }); + + describe('against the driver', () => { + let real; + + before(requireMongo); + + beforeEach(async () => { + clock.restore(); + databaseHealthService.stop(); + real = new MongoClient(`mongodb://${config.database.url}:${config.database.port}`, { socketTimeoutMS: 100 }); + await real.connect(); + databaseHealthService.watch(real, 'probe'); + }); + + afterEach(async () => { + await real.close(); + }); + + it('counts one stuck read as two round trips, so two stuck reads hold the node', async function stuckReads() { + this.timeout(10000); + const ranOut = sinon.spy(log, 'warn'); + const collection = real.db(config.database.local.database).collection('databasehealthprobe'); + await collection.insertOne({ probe: 1 }); + const stuckRead = () => collection.findOne({ $where: 'sleep(400) || true' }).catch((error) => error); + + const first = await stuckRead(); + // the canary: the read did run out, and the driver retried it once + expect(first.name).to.equal('MongoNetworkTimeoutError'); + expect(ranOut.getCalls().filter((call) => /round trip ran out/.test(call.args[0]))).to.have.length(2); + expect(fluxNetworkHelper.isPlacementHeld()).to.equal(false); + + await stuckRead(); + + expect(fluxNetworkHelper.isPlacementHeld()).to.equal(true); + // the server's own sleeps end before the collection is dropped + await new Promise((resolve) => { setTimeout(resolve, 450); }); + await collection.drop(); + }); + }); +}); diff --git a/tests/unit/dbHelper.test.js b/tests/unit/dbHelper.test.js index f08d795b02..f9b2dd4a1b 100644 --- a/tests/unit/dbHelper.test.js +++ b/tests/unit/dbHelper.test.js @@ -1,6 +1,7 @@ const { expect } = require('chai'); const config = require('config'); -const { ObjectId } = require('mongodb'); +const { ObjectId, MongoClient } = require('mongodb'); +const sinon = require('sinon'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const { requireMongo } = require('./dbTestHelper'); @@ -778,6 +779,98 @@ describe('dbHelper tests', () => { }); }); + describe('the maintenance client', () => { + afterEach(async () => { + sinon.restore(); + await dbHelper.closeDbConnection(); + }); + + const client = () => ({ close: sinon.stub().resolves(), db: sinon.stub().callsFake((name) => ({ databaseName: name, client: 'maintenance' })) }); + + it('is connected beside the main client, each bounded by its own configured call timeout', async () => { + await dbHelper.closeDbConnection(); + const main = client(); + const maintenance = client(); + const connect = sinon.stub(MongoClient, 'connect'); + connect.onFirstCall().resolves(main); + connect.onSecondCall().resolves(maintenance); + + await dbHelper.initiateDB(); + + const { callTimeoutMs, maintenanceCallTimeoutMs } = config.database; + expect(callTimeoutMs, 'the two clients share one bound, so this proves nothing').to.not.equal(maintenanceCallTimeoutMs); + expect(connect.firstCall.args[1]).to.include({ maxPoolSize: 100, socketTimeoutMS: callTimeoutMs, waitQueueTimeoutMS: callTimeoutMs }); + expect(connect.secondCall.args[1]).to.include({ socketTimeoutMS: maintenanceCallTimeoutMs, waitQueueTimeoutMS: maintenanceCallTimeoutMs }); + expect(dbHelper.databaseConnection()).to.equal(main); + expect(dbHelper.maintenanceConnection()).to.equal(maintenance); + expect(dbHelper.onMaintenanceClient({ databaseName: 'globalzelappstest' })).to.deep.equal({ databaseName: 'globalzelappstest', client: 'maintenance' }); + }); + + it('is connected with the main client or not at all', async () => { + await dbHelper.closeDbConnection(); + const main = client(); + const connect = sinon.stub(MongoClient, 'connect'); + connect.onFirstCall().resolves(main); + connect.onSecondCall().rejects(new Error('connection refused')); + + const error = await dbHelper.initiateDB().catch((caught) => caught); + + expect(error.message).to.equal('connection refused'); + + sinon.assert.calledOnce(main.close); + expect(dbHelper.databaseConnection()).to.equal(null); + expect(dbHelper.maintenanceConnection()).to.equal(null); + }); + }); + + describe('repairNanInAppsMessagesDb', () => { + const { appsglobal, daemon, local } = config.database; + let messages; + let hashes; + let tracker; + + beforeEach(async () => { + await dbHelper.initiateDB(); + const client = dbHelper.databaseConnection(); + messages = client.db(appsglobal.database).collection(appsglobal.collections.appsMessages); + hashes = client.db(daemon.database).collection(daemon.collections.appsHashes); + tracker = client.db(local.database).collection(local.collections.nodeStartupTracker); + await messages.deleteMany({}); + await hashes.deleteMany({}); + await tracker.deleteMany({}); + await hashes.insertMany([{ hash: 'first', value: 5 }, { hash: 'later', value: 7 }]); + }); + + it('repairs once, and later calls do not scan again unless forced', async () => { + await messages.insertOne({ hash: 'first', valueSat: NaN }); + + await dbHelper.repairNanInAppsMessagesDb(); + expect((await messages.findOne({ hash: 'first' })).valueSat).to.equal(5); + expect(await tracker.findOne({ _id: 'appsMessagesValueSatRepaired' })).to.not.equal(null); + + await messages.insertOne({ hash: 'later', valueSat: NaN }); + await dbHelper.repairNanInAppsMessagesDb(); + expect((await messages.findOne({ hash: 'later' })).valueSat).to.be.NaN; + + await dbHelper.repairNanInAppsMessagesDb({ force: true }); + expect((await messages.findOne({ hash: 'later' })).valueSat).to.equal(7); + }); + + it('leaves no marker when the repair fails', async () => { + await messages.insertOne({ hash: 'first', valueSat: NaN }); + const globalDb = dbHelper.databaseConnection().db(appsglobal.database); + // the repaired value is refused, so the repair's write fails + await globalDb.command({ collMod: appsglobal.collections.appsMessages, validator: { valueSat: { $ne: 5 } } }); + try { + const error = await dbHelper.repairNanInAppsMessagesDb().catch((caught) => caught); + expect(error).to.be.an('error'); + } finally { + await globalDb.command({ collMod: appsglobal.collections.appsMessages, validator: {} }); + } + expect(await tracker.findOne({ _id: 'appsMessagesValueSatRepaired' })).to.equal(null); + }); + }); + describe('rebuilding globalAppsInformation from the message log', () => { const { appsglobal, appslocal, daemon } = config.database; const scannedHeight = 3008000; @@ -836,6 +929,15 @@ describe('dbHelper tests', () => { expect(apps.map((a) => `${a.name}:${a.hash}`).sort()).to.deep.equal(['IncidentApp:i4', 'LegacyApp:l3', 'RenewedApp:r2', 'TiedApp:ta']); }); + it('does not report a registry valid when there is nothing to validate it against', async () => { + await appsGlobalDb.collection(appsglobal.collections.appsMessages).deleteMany({}); + + const result = await dbHelper.validateAppsInformation(); + + expect(result.validated).to.equal(false); + expect(result.reindexed).to.equal(false); + }); + it('should judge the rebuilt collection as valid, not reindex it on every check', async () => { const first = await dbHelper.validateAppsInformation(); expect(first.reindexed).to.equal(true); diff --git a/tests/unit/dbHelperIndexes.test.js b/tests/unit/dbHelperIndexes.test.js new file mode 100644 index 0000000000..0f562076d4 --- /dev/null +++ b/tests/unit/dbHelperIndexes.test.js @@ -0,0 +1,314 @@ +// ensureIndex is the boot sequence's self-healing index builder. It runs in +// serviceManager's prepareDatabaseSchema, which boot runs again until it succeeds: the +// recoverable failures (an options conflict, or a unique build blocked by +// duplicate rows) are healed here, and anything else rethrows - a transient +// blip heals on the next pass, a genuinely broken DB wedges loudly. dedupeByKey is the recovery strategy a unique build +// passes when its key is the row's identity. + +const chai = require('chai'); +const chaiAsPromised = require('chai-as-promised'); +const sinon = require('sinon'); + +chai.use(chaiAsPromised); +const { expect } = chai; + +const { ensureIndex, ensureIndexes, dedupeByKey } = require('../../ZelBack/src/services/dbHelper'); +const log = require('../../ZelBack/src/lib/log'); + +describe('dbHelper ensureIndex', () => { + let logErrorSpy; + + beforeEach(() => { + logErrorSpy = sinon.spy(log, 'error'); + }); + + afterEach(() => { + sinon.restore(); + }); + + function stubCollection(createIndex) { + return { + collectionName: 'somecollection', + createIndex, + listIndexes: sinon.stub(), + dropIndex: sinon.stub().resolves(), + }; + } + + const duplicateError = () => { + const err = new Error('E11000 duplicate key error collection'); + err.code = 11000; + err.codeName = 'DuplicateKey'; + return err; + }; + + it('builds the index and touches nothing else on success', async () => { + const createIndex = sinon.stub().resolves('spec_1'); + const collection = stubCollection(createIndex); + + await ensureIndex(collection, { spec: 1 }, { unique: true }); + + sinon.assert.calledOnceWithExactly(createIndex, { spec: 1 }, { unique: true }); + sinon.assert.notCalled(collection.dropIndex); + sinon.assert.notCalled(logErrorSpy); + }); + + it('drops a conflicting index by its actual name and recreates', async () => { + const conflict = new Error('Index with name: spec_1 already exists with different options'); + conflict.codeName = 'IndexOptionsConflict'; + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(conflict); + createIndex.onSecondCall().resolves('spec_1'); + const collection = stubCollection(createIndex); + collection.listIndexes.returns({ + toArray: async () => [ + { key: { _id: 1 }, name: '_id_' }, + { key: { spec: 1 }, name: 'oldname' }, + ], + }); + + await ensureIndex(collection, { spec: 1 }, { expireAfterSeconds: 60 }); + + sinon.assert.calledOnceWithExactly(collection.dropIndex, 'oldname'); + sinon.assert.calledTwice(createIndex); + sinon.assert.notCalled(logErrorSpy); + }); + + it('drops the index of the same name when its key differs, and recreates', async () => { + const conflict = new Error('An existing index has the same name as the requested index but a different key'); + conflict.codeName = 'IndexKeySpecsConflict'; + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(conflict); + createIndex.onSecondCall().resolves('by image'); + const collection = stubCollection(createIndex); + collection.listIndexes.returns({ + toArray: async () => [ + { key: { _id: 1 }, name: '_id_' }, + { key: { repotag: 1 }, name: 'by image' }, + ], + }); + + await ensureIndex(collection, { 'compose.repotag': 1 }, { name: 'by image' }); + + sinon.assert.calledOnceWithExactly(collection.dropIndex, 'by image'); + sinon.assert.calledTwice(createIndex); + }); + + // A node holding the old non-unique index of the same key also holds the rows it + // never refused: the build after the drop meets them, and repairs them the same way. + it('repairs duplicate rows that block the build after a conflicting index is dropped', async () => { + const conflict = new Error('Index with name: hash_1 already exists with different options'); + conflict.codeName = 'IndexOptionsConflict'; + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(conflict); + createIndex.onSecondCall().rejects(duplicateError()); + createIndex.onThirdCall().resolves('hash_1'); + const collection = stubCollection(createIndex); + collection.listIndexes.returns({ toArray: async () => [{ key: { hash: 1 }, name: 'hash_1' }] }); + const recover = sinon.stub().resolves(2); + + await ensureIndex(collection, { hash: 1 }, { unique: true }, recover); + + sinon.assert.calledOnceWithExactly(collection.dropIndex, 'hash_1'); + sinon.assert.calledOnceWithExactly(recover, collection, { hash: 1 }, { unique: true }); + sinon.assert.calledThrice(createIndex); + }); + + it('runs the recovery strategy and rebuilds when duplicate rows block a unique index', async () => { + // The node ends up WITH the index, not running degraded without it. + const createIndex = sinon.stub(); + createIndex.onFirstCall().rejects(duplicateError()); + createIndex.onSecondCall().resolves('hash_1'); + const collection = stubCollection(createIndex); + const recover = sinon.stub().resolves(3); + + await ensureIndex(collection, { hash: 1 }, { unique: true }, recover); + + sinon.assert.calledOnceWithExactly(recover, collection, { hash: 1 }, { unique: true }); + sinon.assert.calledTwice(createIndex); + // recover ran before the rebuild + sinon.assert.callOrder(createIndex, recover, createIndex); + }); + + it('rethrows a duplicate-key failure with no recovery strategy, rather than skipping', async () => { + // A unique build with no strategy is not silently swallowed - it must be + // seen, not hidden until the next reboot. + const createIndex = sinon.stub().rejects(duplicateError()); + const collection = stubCollection(createIndex); + + await expect(ensureIndex(collection, { hash: 1 }, { unique: true })).to.be.rejectedWith('E11000'); + sinon.assert.calledOnce(createIndex); + }); + + it('rethrows a transient failure so the boot retry can heal it', async () => { + // Not swallowed: index setup runs before services start, so the retry loop + // re-runs safely and a transient blip heals rather than being skipped. + const transient = new Error('connection 5 to mongo timed out'); + transient.codeName = 'NetworkTimeout'; + const createIndex = sinon.stub().rejects(transient); + const collection = stubCollection(createIndex); + const recover = sinon.stub().resolves(0); + + await expect(ensureIndex(collection, { spec: 1 }, {}, recover)).to.be.rejectedWith('timed out'); + // recovery is only for the duplicate-key case, not for a transient error + sinon.assert.notCalled(recover); + }); + + it('rethrows if the rebuild after recovery still fails', async () => { + const createIndex = sinon.stub().rejects(duplicateError()); + const collection = stubCollection(createIndex); + const recover = sinon.stub().resolves(1); + + await expect(ensureIndex(collection, { hash: 1 }, { unique: true }, recover)).to.be.rejectedWith('E11000'); + sinon.assert.calledOnce(recover); + sinon.assert.calledTwice(createIndex); + }); +}); + +describe('dbHelper dedupeByKey', () => { + function stubCollection(groups) { + return { + collectionName: 'somecollection', + aggregate: sinon.stub().returns({ toArray: async () => groups }), + deleteMany: sinon.stub().resolves({ deletedCount: 0 }), + }; + } + + it('removes all but the newest per key group and reports the count', async () => { + // ids come newest-first from the pipeline's _id sort, so slice(1) keeps the + // newest and removes the rest. + const collection = stubCollection([ + { _id: { k0: 'a' }, ids: ['new1', 'old1', 'old2'] }, + { _id: { k0: 'b' }, ids: ['new3', 'old3'] }, + ]); + + const removed = await dedupeByKey(collection, { hash: 1 }, { unique: true }); + + expect(removed).to.equal(3); + sinon.assert.calledOnceWithExactly(collection.deleteMany, { _id: { $in: ['old1', 'old2', 'old3'] } }); + }); + + it('builds the group key from every field of the index spec', async () => { + const collection = stubCollection([]); + + await dedupeByKey(collection, { 'data.name': 1, 'data.ip': 1 }, {}); + + const [pipeline] = collection.aggregate.firstCall.args; + const group = pipeline.find((stage) => stage.$group); + expect(group.$group._id).to.deep.equal({ k0: '$data.name', k1: '$data.ip' }); + }); + + it('honours the index partial filter so it only touches covered rows', async () => { + const collection = stubCollection([]); + const partial = { incidentKey: { $exists: true } }; + + await dedupeByKey(collection, { appName: 1, eventType: 1, incidentKey: 1 }, { partialFilterExpression: partial }); + + const [pipeline] = collection.aggregate.firstCall.args; + expect(pipeline[0]).to.deep.equal({ $match: partial }); + }); + + it('deletes nothing when there are no duplicates', async () => { + const collection = stubCollection([]); + + const removed = await dedupeByKey(collection, { hash: 1 }, {}); + + expect(removed).to.equal(0); + sinon.assert.notCalled(collection.deleteMany); + }); +}); + +// ensureIndexes is the same guarantee taken in one command. mongo pays a fixed +// cost per index BUILD whatever the data size, so a node asserting its schema +// one index at a time pays it 34 times over 14 collections - and in the +// integration harness, where ten nodes share one mongod and every database is +// new, ten nodes do it at once (measured: 938 builds per fleet boot). The batch +// is the fast path only: anything that fails falls back to ensureIndex, because +// the healing above has to know WHICH index failed and a batch rejection does +// not say. +describe('dbHelper ensureIndexes', () => { + afterEach(() => { + sinon.restore(); + }); + + function stubCollection({ createIndexes, createIndex } = {}) { + return { + collectionName: 'somecollection', + createIndexes: createIndexes || sinon.stub().resolves(['a_1']), + createIndex: createIndex || sinon.stub().resolves('a_1'), + listIndexes: sinon.stub(), + dropIndex: sinon.stub().resolves(), + }; + } + + it('asserts every index in ONE command, not one command per index', async () => { + const collection = stubCollection(); + + await ensureIndexes(collection, [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'by name' }, + { key: { ip: 1, name: 1 } }, + ]); + + sinon.assert.calledOnce(collection.createIndexes); + sinon.assert.notCalled(collection.createIndex); + expect(collection.createIndexes.firstCall.args[0]).to.deep.equal([ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { name: 1 }, name: 'by name' }, + { key: { ip: 1, name: 1 } }, + ]); + }); + + it('never hands mongo the recovery strategy, which is ours', async () => { + const collection = stubCollection(); + + await ensureIndexes(collection, [ + { key: { hash: 1 }, unique: true, recover: dedupeByKey }, + ]); + + const [models] = collection.createIndexes.firstCall.args; + expect(models).to.deep.equal([{ key: { hash: 1 }, unique: true }]); + expect(models[0]).to.not.have.property('recover'); + }); + + it('falls back to one at a time when the batch is refused, and still heals', async () => { + const duplicate = new Error('E11000 duplicate key error collection'); + duplicate.code = 11000; + duplicate.codeName = 'DuplicateKey'; + + const createIndex = sinon.stub(); + createIndex.onFirstCall().resolves('expireAt_1'); + createIndex.onSecondCall().rejects(duplicate); + createIndex.onThirdCall().resolves('hash_1'); + + const collection = stubCollection({ + createIndexes: sinon.stub().rejects(duplicate), + createIndex, + }); + const recover = sinon.stub().resolves(2); + + await ensureIndexes(collection, [ + { key: { expireAt: 1 }, expireAfterSeconds: 0 }, + { key: { hash: 1 }, unique: true, recover }, + ]); + + // one per index, with the recovery run between the failed build and its retry + sinon.assert.calledThrice(createIndex); + sinon.assert.calledOnce(recover); + sinon.assert.calledWithExactly(createIndex.firstCall, { expireAt: 1 }, { expireAfterSeconds: 0 }); + sinon.assert.calledWithExactly(createIndex.secondCall, { hash: 1 }, { unique: true }); + sinon.assert.calledWithExactly(createIndex.thirdCall, { hash: 1 }, { unique: true }); + }); + + it('still rethrows what the per-index path cannot heal', async () => { + const fatal = new Error('not authorized on db to execute command'); + fatal.codeName = 'Unauthorized'; + + const collection = stubCollection({ + createIndexes: sinon.stub().rejects(fatal), + createIndex: sinon.stub().rejects(fatal), + }); + + await expect(ensureIndexes(collection, [{ key: { a: 1 } }])).to.be.rejectedWith('not authorized'); + }); +}); diff --git a/tests/unit/dockerService.test.js b/tests/unit/dockerService.test.js index 5348a7fe94..0c819fd768 100644 --- a/tests/unit/dockerService.test.js +++ b/tests/unit/dockerService.test.js @@ -2330,6 +2330,20 @@ describe('dockerService tests', () => { await expect(dockerService.appDockerCreate(nodeApp, appName, true)).to.eventually.be.rejectedWith('Cannot read properties of undefined (reading \'forEach\')'); }); + + describe('the app\'s slice', () => { + const nodeApp = { + ...baseNodeApp, containerPorts: [], ports: [], version: 3, + }; + const created = () => dockerStub.firstCall.args[0]; + it('puts the container in its app\'s slice', async () => { + sinon.stub(Dockerode.prototype, 'info').resolves({ CgroupDriver: 'systemd', DriverStatus: [] }); + + await dockerService.appDockerCreate(nodeApp, 'my-app', true); + + expect(created().HostConfig.CgroupParent).to.equal('flux-my\\x2dapp.slice'); + }); + }); }); describe('loadImage tests', () => { const ID = 'sha256:1111111111111111111111111111111111111111111111111111111111111111'; diff --git a/tests/unit/enterpriseHelper.test.js b/tests/unit/enterpriseHelper.test.js new file mode 100644 index 0000000000..077fe83a39 --- /dev/null +++ b/tests/unit/enterpriseHelper.test.js @@ -0,0 +1,52 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +const MODULE_PATH = '../../ZelBack/src/services/utils/enterpriseHelper'; + +// isArcane is read when the module loads, so FLUXOS_PATH is set around the load. Reaching the +// database is the first step of a decrypt; the stub throws so a test sees whether it was reached. +function load() { + const previous = process.env.FLUXOS_PATH; + process.env.FLUXOS_PATH = '/tmp/fluxos'; + const dbHelper = { databaseConnection: sinon.stub().throws(new Error('decrypt reached')) }; + try { + const m = proxyquire(MODULE_PATH, { + '../dbHelper': dbHelper, + '../benchmarkService': {}, + '../../lib/log': { info: sinon.stub(), warn: sinon.stub(), error: sinon.stub() }, + }); + return { module: m, dbHelper }; + } finally { + if (previous === undefined) delete process.env.FLUXOS_PATH; + else process.env.FLUXOS_PATH = previous; + } +} + +describe('enterpriseHelper', () => { + afterEach(() => sinon.restore()); + + describe('checkAndDecryptAppSpecs', () => { + // A spec is enterprise exactly when version >= 8 and it carries an enterprise blob; the + // route checks use the same rule, so every spec that reaches a decrypt has passed them. + it('decrypts a version 8 spec with an enterprise blob', async () => { + const { module: m, dbHelper } = load(); + await m.checkAndDecryptAppSpecs({ version: 8, name: 'app', enterprise: 'blob' }) + .then(() => expect.fail('resolved without a decrypt'), (error) => expect(error.message).to.equal('decrypt reached')); + sinon.assert.calledOnce(dbHelper.databaseConnection); + }); + + [ + ['no version', { name: 'app', enterprise: 'blob' }], + ['a non-numeric version', { version: 'eight', name: 'app', enterprise: 'blob' }], + ['version 7', { version: 7, name: 'app', enterprise: 'blob' }], + ['an empty enterprise', { version: 8, name: 'app', enterprise: '' }], + ].forEach(([label, spec]) => { + it(`returns a spec with ${label} unchanged, without a decrypt`, async () => { + const { module: m, dbHelper } = load(); + expect(await m.checkAndDecryptAppSpecs(spec)).to.equal(spec); + sinon.assert.notCalled(dbHelper.databaseConnection); + }); + }); + }); +}); diff --git a/tests/unit/explorerService.test.js b/tests/unit/explorerService.test.js index 482e03d41d..fd0357b494 100644 --- a/tests/unit/explorerService.test.js +++ b/tests/unit/explorerService.test.js @@ -11,9 +11,13 @@ const daemonServiceAddressRpcs = require('../../ZelBack/src/services/daemonServi const daemonServiceMiscRpcs = require('../../ZelBack/src/services/daemonService/daemonServiceMiscRpcs'); const daemonServiceUtils = require('../../ZelBack/src/services/daemonService/daemonServiceUtils'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); +const { MongoNetworkTimeoutError } = require('mongodb'); const globalState = require('../../ZelBack/src/services/utils/globalState'); const log = require('../../ZelBack/src/lib/log'); const { requireMongo } = require('./dbTestHelper'); +const { UNPOSITIONED } = require('../../ZelBack/src/services/migrations/paymentPositions'); +const messageVerifier = require('../../ZelBack/src/services/appMessaging/messageVerifier'); const chai = require('chai'); const chaiAsPromised = require('chai-as-promised'); @@ -1210,6 +1214,17 @@ describe('explorerService tests', () => { sinon.assert.calledWithMatch(updateInDatabaseStub, sinon.match.object, 'addresstransactionindex', {}, { $pull: { transactions: { height: sinon.match.object } } }); }); + it('runs its deletes and updates on the maintenance client', async () => { + removeDocumentsFromCollectionStub.returns(true); + updateInDatabaseStub.returns(true); + + await explorerService.restoreDatabaseToBlockheightState(100000); + + const clients = [...removeDocumentsFromCollectionStub.getCalls(), ...updateInDatabaseStub.getCalls()].map((call) => call.args[0].client); + expect(clients).to.have.length.above(0); + clients.forEach((client) => expect(client).to.equal(dbHelper.maintenanceConnection())); + }); + it('should leave the app messages and the registry alone', async () => { removeDocumentsFromCollectionStub.returns(true); updateInDatabaseStub.returns(true); @@ -1988,7 +2003,6 @@ describe('explorerService tests', () => { sinon.assert.notCalled(logErrorSpy); sinon.assert.calledWithMatch(logInfoSpy, 'Bootstrap: Using address-index fast path'); sinon.assert.calledWithMatch(logInfoSpy, 'Bootstrap complete'); - sinon.assert.calledWithMatch(logInfoSpy, 'Preparing apps collections'); sinon.assert.calledWithMatch(logInfoSpy, 'Preparation done'); }); @@ -2096,7 +2110,6 @@ describe('explorerService tests', () => { sinon.assert.notCalled(logErrorSpy); sinon.assert.calledWithMatch(logInfoSpy, 'Bootstrap: Using address-index fast path'); - sinon.assert.calledWithMatch(logInfoSpy, 'Preparing apps collections'); sinon.assert.calledWithMatch(logInfoSpy, 'Preparation done'); const dropped = dropCollectionStub.getCalls().map((call) => call.args[1]); expect(dropped).to.not.include('zelappslocation'); @@ -2108,6 +2121,137 @@ describe('explorerService tests', () => { }); }); + describe('the explorer\'s collections', () => { + const { appsHashes } = config.database.daemon.collections; + let hashes; + + beforeEach(async () => { + await dbHelper.initiateDB(); + hashes = dbHelper.databaseConnection().db(config.database.daemon.database).collection(appsHashes); + await hashes.drop().catch(() => {}); + }); + + afterEach(async () => { + sinon.restore(); + await hashes.drop().catch(() => {}); + }); + + it('keeps, of each hash\'s payment records, the earliest whose transaction is on the chain', async () => { + await hashes.insertMany([ + { hash: 'paid', txid: 'stale', height: 200, notOnChain: true }, + { hash: 'paid', txid: 'later', height: 400, txIndex: 1 }, + { hash: 'paid', txid: 'first', height: 300, txIndex: 5 }, + { hash: 'paid', txid: 'sameblock', height: 300, txIndex: 7 }, + { hash: 'single', txid: 'only', height: 500 }, + ]); + + const removed = await explorerService.keepEarliestPayment(hashes); + + expect(removed).to.equal(3); + expect((await hashes.find({}, { projection: { _id: 0, txid: 1 } }).sort({ txid: 1 }).toArray()).map((r) => r.txid)).to.deep.equal(['first', 'only']); + }); + + it('keeps every duplicate when the stored message cannot be moved, so the next index build recovers them again', async () => { + await hashes.insertMany([ + { hash: 'paid', txid: 'first', height: 300 }, + { hash: 'paid', txid: 'later', height: 400 }, + ]); + sinon.stub(messageVerifier, 'moveStoredMessagesToPayments').rejects(new Error('connection timed out')); + + await expect(explorerService.keepEarliestPayment(hashes)).to.be.rejectedWith('connection timed out'); + + expect(await hashes.countDocuments({ hash: 'paid' })).to.equal(2); + }); + + it('moves the stored message of a hash it kept the earliest payment of onto that payment', async () => { + const messages = dbHelper.databaseConnection().db(config.database.appsglobal.database).collection(config.database.appsglobal.collections.appsMessages); + await messages.deleteMany({ hash: { $in: ['paid', 'single'] } }); + await hashes.insertMany([ + { hash: 'paid', txid: 'first', height: 300, value: 5 }, + { hash: 'paid', txid: 'later', height: 400, value: 7 }, + { hash: 'single', txid: 'only', height: 500, value: 9 }, + ]); + await messages.insertMany([ + { hash: 'paid', txid: 'later', height: 400, valueSat: 7, appSpecifications: { name: 'PaidTwice' } }, + { hash: 'single', txid: 'only', height: 500, valueSat: 9, appSpecifications: { name: 'PaidOnce' } }, + ]); + try { + await explorerService.keepEarliestPayment(hashes); + + const paid = await messages.findOne({ hash: 'paid' }, { projection: { _id: 0, txid: 1, height: 1, valueSat: 1 } }); + expect(paid).to.deep.equal({ txid: 'first', height: 300, valueSat: 5 }); + const single = await messages.findOne({ hash: 'single' }, { projection: { _id: 0, txid: 1, height: 1 } }); + expect(single, 'a hash with one payment is left as it is').to.deep.equal({ txid: 'only', height: 500 }); + } finally { + await messages.deleteMany({ hash: { $in: ['paid', 'single'] } }); + } + }); + + it('asserts every index on the maintenance client', async () => { + const ensureIndexes = sinon.stub(dbHelper, 'ensureIndexes').resolves(); + + await explorerService.prepareCollections(); + + expect(dbHelper.maintenanceConnection(), 'no maintenance client, so this proves nothing').to.not.equal(null); + expect(ensureIndexes.callCount).to.equal(5); + ensureIndexes.getCalls().forEach((call) => expect(call.args[0].client).to.equal(dbHelper.maintenanceConnection())); + }); + + it('builds the unique hash index over records that hold duplicates, keeping the earliest payment', async () => { + // inserted earliest first, so the newest row is the later payment + await hashes.insertMany([ + { hash: 'paid', txid: 'first', height: 300 }, + { hash: 'paid', txid: 'later', height: 400 }, + ]); + + await explorerService.prepareCollections(); + + const unique = (await hashes.indexes()).find((index) => index.name === 'query for getting app hash'); + expect(unique?.unique, 'the unique hash index').to.equal(true); + expect((await hashes.find({ hash: 'paid' }).toArray()).map((r) => r.txid)).to.deep.equal(['first']); + }); + + it('lets the boot backfill read only the payment records without a position', async () => { + await hashes.insertMany([ + ...Array.from({ length: 50 }, (_, i) => ({ hash: `placed${i}`, txid: `placed${i}`, height: i, txIndex: 0 })), + { hash: 'unplaced', txid: 'unplaced', height: 60 }, + ]); + + await explorerService.prepareCollections(); + + const { executionStats } = await hashes.find(UNPOSITIONED).explain('executionStats'); + expect(executionStats.nReturned).to.equal(1); + expect(executionStats.totalDocsExamined, 'the backfill read records that have a position').to.equal(1); + }); + + it('starts no scan from zero until its collections hold their indexes', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves({ generalScannedHeight: 0 }); + sinon.stub(dbHelper, 'countInDatabase').resolves(1); + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true } }); + sinon.stub(daemonServiceBlockchainRpcs, 'getBlockCount').returns({ status: 'success', data: 200000 }); + // a scan that starts stops the processor, so the assertion below reads it + const scan = sinon.stub(daemonServiceBlockchainRpcs, 'getBlock').callsFake(async () => { + explorerService.setBlockProccessingCanContinue(false); + return { status: 'error', data: {} }; + }); + sinon.stub(dbHelper, 'ensureIndexes').rejects(new MongoNetworkTimeoutError('timed out')); + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); + explorerService.setZelAppSpecsMigrationDone(true); + explorerService.setBlockProccessingCanContinue(true); + try { + await explorerService.initiateBlockProcessor(false, false); + await clock.tickAsync(1000); + + sinon.assert.notCalled(scan); + } finally { + explorerService.setBlockProccessingCanContinue(false); + explorerService.setIsInInitiationOfBP(false); + explorerService.setZelAppSpecsMigrationDone(false); + clock.restore(); + } + }); + }); + describe('getAddressUtxos tests', () => { let findInDatabaseStub; let logErrorSpy; @@ -2676,4 +2820,179 @@ describe('explorerService tests', () => { sinon.assert.notCalled(updateOneStub); }); }); + + describe('a chain write that fails', () => { + // A soft fork the foundation sent to itself: a price fork every node applies. + const softForkBlock = () => ({ + height: 1000, + tx: [{ + version: 4, + txid: 'softforktx', + vin: [{ address: config.fluxapps.addressMultisig }], + vout: [{ + valueSat: 0, + scriptPubKey: { + addresses: [config.fluxapps.addressMultisig], + asm: `OP_RETURN ${Buffer.from('p_1_2_3_4').toString('hex')}`, + }, + }], + }], + }); + + beforeEach(() => { + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(log, 'error'); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('fails the block when its soft fork cannot be written', async () => { + sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new MongoNetworkTimeoutError('connection timed out')); + + const outcome = await explorerService.processInsight(softForkBlock(), {}).catch((err) => err); + + expect(outcome, 'the block moved on without its price fork').to.be.instanceOf(MongoNetworkTimeoutError); + }); + + it('a canary: a soft fork that fails for another reason is skipped', async () => { + const updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new Error('unreadable message')); + + await explorerService.processInsight(softForkBlock(), {}); + + sinon.assert.called(updateOne); + }); + + describe('on a node without the insight index', () => { + // The same soft fork, its sender found through the spent output it names. + const standardSoftForkBlock = () => ({ + ...softForkBlock(), + tx: softForkBlock().tx.map((tx) => ({ ...tx, vin: [{ txid: 'fundingtx', vout: 0 }] })), + }); + const { chainMessages } = config.database.chainparams.collections; + let updateOne; + + beforeEach(() => { + sinon.stub(dbHelper, 'findOneAndDeleteInDatabase').resolves({ value: { address: config.fluxapps.addressMultisig } }); + sinon.stub(dbHelper, 'insertOneToDatabase').resolves(); + sinon.stub(serviceHelper, 'delay').resolves(); + // The block's address index is written the same way first; only the soft + // fork's own write fails. + updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').resolves(); + }); + + it('fails the block when its soft fork cannot be written', async () => { + updateOne.withArgs(sinon.match.any, chainMessages).rejects(new MongoNetworkTimeoutError('connection timed out')); + + const outcome = await explorerService.processStandard(standardSoftForkBlock(), {}).catch((err) => err); + + sinon.assert.calledWith(updateOne, sinon.match.any, chainMessages); + expect(outcome, 'the block moved on without its price fork').to.be.instanceOf(MongoNetworkTimeoutError); + }); + + it('a canary: a soft fork that fails for another reason is skipped', async () => { + updateOne.withArgs(sinon.match.any, chainMessages).rejects(new Error('unreadable message')); + + await explorerService.processStandard(standardSoftForkBlock(), {}); + + sinon.assert.calledWith(updateOne, sinon.match.any, chainMessages); + }); + }); + + it('fails the block when one of its app payments cannot be recorded', async () => { + sinon.stub(dbHelper, 'insertManyToDatabase').rejects(new Error('batch failed')); + const upsert = sinon.stub(dbHelper, 'updateOneInDatabase'); + upsert.onFirstCall().resolves(); + upsert.onSecondCall().rejects(new MongoNetworkTimeoutError('connection timed out')); + + const outcome = await explorerService.insertTransactions([{ hash: 'a', height: 1 }, { hash: 'b', height: 1 }], {}).catch((err) => err); + + expect(outcome, 'the block moved on with a payment unrecorded').to.be.instanceOf(MongoNetworkTimeoutError); + sinon.assert.calledTwice(upsert); + }); + }); + + describe('explorer reset requests', () => { + let res; + let clock; + + beforeEach(() => { + res = { json: sinon.spy(), status: sinon.stub().returnsThis() }; + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: sinon.fake.returns({}) }); + sinon.stub(log, 'error'); + explorerService.setIsInInitiationOfBP(false); + explorerService.setBlockProccessingCanContinue(true); + }); + + afterEach(() => { + clock?.restore(); + clock = undefined; + explorerService.setIsInInitiationOfBP(false); + explorerService.setBlockProccessingCanContinue(true); + sinon.restore(); + }); + + async function settle() { + for (let k = 0; k < 5; k += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } + } + + it('reindex answers once with the error and releases the operation when the drop fails', async () => { + const dropCollection = sinon.stub(dbHelper, 'dropCollection').rejects(new Error('connection timed out')); + + await explorerService.reindexExplorer({ params: {} }, res); + await settle(); + + sinon.assert.calledOnce(res.json); + sinon.assert.calledWithMatch(res.json, { status: 'error', data: { message: 'connection timed out' } }); + + dropCollection.rejects(new Error('connection timed out again')); + res.json.resetHistory(); + await explorerService.reindexExplorer({ params: {} }, res); + await settle(); + sinon.assert.calledWithMatch(res.json, { data: { message: 'connection timed out again' } }); + }); + + it('rescan answers once with the error and releases the operation when the height write fails', async () => { + sinon.stub(dbHelper, 'findOneInDatabase').resolves({ generalScannedHeight: 2000 }); + const updateOne = sinon.stub(dbHelper, 'updateOneInDatabase').rejects(new Error('connection timed out')); + + await explorerService.rescanExplorer({ params: { blockheight: 1000 } }, res); + await settle(); + + sinon.assert.calledOnce(res.json); + sinon.assert.calledWithMatch(res.json, { status: 'error', data: { message: 'connection timed out' } }); + + updateOne.rejects(new Error('connection timed out again')); + res.json.resetHistory(); + await explorerService.rescanExplorer({ params: { blockheight: 1000 } }, res); + await settle(); + sinon.assert.calledWithMatch(res.json, { data: { message: 'connection timed out again' } }); + }); + + it('rescan without a block height answers once', async () => { + const findOne = sinon.stub(dbHelper, 'findOneInDatabase'); + + await explorerService.rescanExplorer({ params: {} }, res); + + sinon.assert.calledOnce(res.json); + sinon.assert.calledWithMatch(res.json, { status: 'error', data: { message: 'No blockheight provided' } }); + sinon.assert.notCalled(findOne); + }); + + it('a stop that does not take leaves block processing allowed to continue', async () => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + explorerService.setIsInInitiationOfBP(true); + + await explorerService.stopBlockProcessing({}, res); + await clock.tickAsync(13000); + + sinon.assert.calledWithMatch(res.json, { status: 'error' }); + expect(explorerService.getBlockProccessingCanContinue()).to.equal(true); + }); + }); }); diff --git a/tests/unit/fixtures/globalState.js b/tests/unit/fixtures/globalState.js index e73d4143a4..d8fab64de5 100644 --- a/tests/unit/fixtures/globalState.js +++ b/tests/unit/fixtures/globalState.js @@ -44,7 +44,8 @@ function resetGlobalState() { // so they are emptied that way below rather than written to here. else if (Array.isArray(value)) { if (!Object.isFrozen(value)) value.length = 0; } else if (value && typeof value === 'object' && value.constructor === Object) { - for (const own of Object.keys(value)) delete value[own]; + // An object's functions are its interface, not its state. + for (const own of Object.keys(value)) if (typeof value[own] !== 'function') delete value[own]; } } for (const appname of globalState.backupInProgress) globalState.finishBackup(appname); diff --git a/tests/unit/fluxCommunication.test.js b/tests/unit/fluxCommunication.test.js index fd786e3364..aa0b2aa856 100644 --- a/tests/unit/fluxCommunication.test.js +++ b/tests/unit/fluxCommunication.test.js @@ -3,6 +3,7 @@ const sinon = require('sinon'); const objectHash = require('object-hash'); const WebSocket = require('ws'); const { expect } = require('chai'); +const { MongoNetworkTimeoutError } = require('mongodb'); const log = require('../../ZelBack/src/lib/log'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); const { FluxTTLCache } = require('../../ZelBack/src/services/utils/cacheManager'); @@ -27,6 +28,7 @@ const { peerManager } = require('../../ZelBack/src/services/utils/peerState'); const { PEER_SOURCE } = require('../../ZelBack/src/services/utils/FluxPeerSocket'); const rateLimit = require('../../ZelBack/src/services/utils/rateLimit'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); +const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); let localWsServer; let localWsUrl; @@ -484,6 +486,44 @@ describe('fluxCommunication tests', () => { ).to.equal(false); }); + it('holds a message its database could not verify and dispatches it again, once, later', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); + const data = { + type: 'fluxipchanged', version: 1, oldIP: '1.1.1.3', newIP: '2.2.2.3', broadcastedAt: Date.now(), + }; + verify.onFirstCall().resolves(fluxCommunicationUtils.VerifyResult.UNREADABLE); + + await peerManager.messageDispatcher(envelope(data), peerSocket); + await peerManager.messageDispatcher(envelope(data), peerSocket); + + expect(cacheManager.announcementSeen.has(objectHash(data))).to.equal(true); + expect(verify.callCount, 'a copy waits with it').to.equal(1); + sinon.assert.notCalled(messageStore.storeIPChangedMessage); + + // the handler runs after verification, in its own turn: the store is what + // shows the re-dispatched broadcast was handled + const stored = new Promise((resolve) => { messageStore.storeIPChangedMessage.callsFake(async () => { resolve(); return false; }); }); + await clock.tickAsync(30 * 1000); + await clock.tickAsync(100); + await stored; + + expect(verify.callCount).to.equal(2); + sinon.assert.calledOnce(messageStore.storeIPChangedMessage); + }); + + it('gives the slot back for a message its database could not verify within its window', async () => { + const data = { + type: 'fluxipchanged', version: 1, oldIP: '1.1.1.4', newIP: '2.2.2.4', broadcastedAt: Date.now(), + }; + verify.resolves(fluxCommunicationUtils.VerifyResult.UNREADABLE); + fluxCommunicationUtils.verifyTimestampInFluxBroadcast.returns(false); + + await peerManager.messageDispatcher(envelope(data), peerSocket); + + expect(cacheManager.announcementSeen.has(objectHash(data))).to.equal(false); + expect(peerSocket.badMessageTimestamps).to.deep.equal([]); + }); + // The canary: the same type, verifying, DOES hold it - so the release above is the // verification failing and not the filter having been switched off. it('keeps the slot for a message that verifies', async () => { @@ -513,7 +553,9 @@ describe('fluxCommunication tests', () => { }); it('should broadcast the app message if a proper data is given', async () => { - sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ stored: true, rebroadcast: true }); + sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ + stored: true, rebroadcast: true, written: [{ name: 'myApp', hash: 'myapphash' }], refused: [], cleared: false, + }); sinon.stub(messageStore, 'storeAppStateEvent'); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 0 } }); const fromIp = '127.0.0.5'; @@ -590,6 +632,126 @@ describe('fluxCommunication tests', () => { sinon.assert.notCalled(broadcastHashSpy); }).timeout(5000); + + const runningMessage = () => ({ + data: { + type: 'fluxapprunning', version: 2, apps: [{ name: 'myApp', hash: 'myapphash' }], broadcastedAt: Date.now(), ip: '203.0.113.31:16127', + }, + timestamp: Date.now(), + }); + const apprunningPublished = (publish) => publish.getCalls().filter((call) => call.args[0] === 'network:apprunning'); + + it('publishes nothing for a message that wrote, refused, cleared and relayed nothing', async () => { + sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ + stored: false, rebroadcast: false, written: [], refused: [], cleared: false, + }); + sinon.stub(messageStore, 'storeAppStateEvent'); + const publish = sinon.stub(fluxEventBus, 'publish'); + + await fluxCommunication.handleAppRunningMessage(runningMessage(), '127.0.0.5', '16127'); + + expect(apprunningPublished(publish)).to.have.lengthOf(0); + }); + + it('publishes the names it refused, though it wrote and relayed nothing', async () => { + sinon.stub(messageStore, 'storeAppRunningMessage').resolves({ + stored: false, rebroadcast: false, written: [], refused: ['unhashed'], cleared: false, + }); + sinon.stub(messageStore, 'storeAppStateEvent'); + const publish = sinon.stub(fluxEventBus, 'publish'); + + await fluxCommunication.handleAppRunningMessage(runningMessage(), '127.0.0.5', '16127'); + + const [call] = apprunningPublished(publish); + expect(call.args[1]).to.deep.equal({ + ip: '203.0.113.31:16127', apps: [], refused: ['unhashed'], cleared: false, relayed: false, + }); + }); + + // A refused message is logged where it arrived, and goes no further. + it('logs a message its store refuses, and does not relay it', async () => { + sinon.stub(messageStore, 'storeAppRunningMessage').resolves(new Error('Invalid Flux App Running message for storing')); + sinon.stub(messageStore, 'storeAppStateEvent'); + const warn = sinon.stub(log, 'warn'); + const fromIp = '127.0.0.5'; + const port = '16127'; + const message = { + data: { + type: 'fluxapprunning', version: 2, apps: 'not an array', broadcastedAt: Date.now(), ip: '203.0.113.30:16127', + }, + timestamp: Date.now(), + }; + + await fluxCommunication.handleAppRunningMessage(message, fromIp, port); + + sinon.assert.notCalled(broadcastHashSpy); + sinon.assert.calledWithMatch(warn, /handleAppRunningMessage - not stored, from 127\.0\.0\.5:16127 for 203\.0\.113\.30:16127: Invalid Flux App Running message for storing/); + }); + }); + + describe('the installing claim handlers', () => { + // Each handler records the claim, stores the signed broadcast this node serves + // to peers syncing from it, and relays the claim only once both are recorded. + const handlers = [ + { + name: 'handleAppInstallingMessage', + type: 'fluxappinstalling', + storeClaim: 'storeAppInstallingMessage', + storeSigned: 'storeSignedAppInstallingBroadcast', + }, + { + name: 'handleAppInstallingErrorMessage', + type: 'fluxappinstallingerror', + storeClaim: 'storeAppInstallingErrorMessage', + storeSigned: 'storeSignedAppInstallingErrorBroadcast', + }, + ]; + const fromIp = '127.0.0.5'; + const port = '16127'; + const claim = (type) => ({ + timestamp: Date.now(), + data: { + type, version: 1, name: 'claimedapp', ip: '127.0.0.9:16127', broadcastedAt: Date.now(), + }, + }); + let broadcastHash; + + beforeEach(() => { + broadcastHash = sinon.stub(peerManager, 'broadcastHash'); + sinon.stub(log, 'error'); + }); + + afterEach(() => { + sinon.restore(); + }); + + handlers.forEach(({ + name, type, storeClaim, storeSigned, + }) => { + describe(name, () => { + beforeEach(() => { + sinon.stub(messageStore, storeClaim).resolves(true); + }); + + it('a canary: relays a claim whose signed broadcast is stored', async () => { + sinon.stub(messageStore, storeSigned).resolves(); + const message = claim(type); + + await fluxCommunication[name](message, fromIp, port); + + sinon.assert.calledOnceWithExactly(broadcastHash, objectHash(message.data), `${fromIp}:${port}`); + }); + + it('does not relay a claim whose signed broadcast could not be stored', async () => { + const storeSignedStub = sinon.stub(messageStore, storeSigned).rejects(new MongoNetworkTimeoutError('connection timed out')); + + await fluxCommunication[name](claim(type), fromIp, port); + + sinon.assert.calledOnce(storeSignedStub); + sinon.assert.notCalled(broadcastHash); + }); + }); + }); }); @@ -2200,6 +2362,55 @@ describe('fluxCommunication tests', () => { sinon.assert.notCalled(messageStore.storeIPChangedMessage); }); + // A node's newest broadcast says which of its rows stand; its refused + // entries say nothing about what it runs. + describe('pruning against what a synced node runs', () => { + const apprunning = (apps, broadcastedAt = Date.now() - 60000) => ({ + type: 'apprunning', + ip: SENDER, + envelope, + data: { + type: 'fluxapprunning', version: 2, apps, ip: SENDER, broadcastedAt, osUptime: 1, staticIp: false, + }, + }); + + beforeEach(() => { + sinon.stub(messageStore, 'storeBatchAppRunningMessages').resolves({ stored: 1, writeFailed: false }); + }); + + it('prunes against the apps the broadcast names that can be stored', async () => { + await fluxCommunication.handleAppRunningSyncResponse({ + data: { type: 'fluxapprunningsync', messages: [apprunning([{ name: 'web', hash: 'h1' }, { name: 'probe' }])], done: true }, + }, PEER_SOCKET); + + const [newestByIp] = messageStore.pruneAppRunningLocations.firstCall.args; + expect(newestByIp.get(SENDER).names).to.deep.equal(['web']); + }); + + it('never prunes against a broadcast whose every app is refused', async () => { + await fluxCommunication.handleAppRunningSyncResponse({ + data: { type: 'fluxapprunningsync', messages: [apprunning([{ name: 'probe' }])], done: true }, + }, PEER_SOCKET); + + const [newestByIp] = messageStore.pruneAppRunningLocations.firstCall.args; + expect(newestByIp.has(SENDER), 'pruned the sender\'s rows against an empty list').to.equal(false); + }); + + it('prunes against the newest broadcast it can read, past a later one it refuses', async () => { + const now = Date.now(); + await fluxCommunication.handleAppRunningSyncResponse({ + data: { + type: 'fluxapprunningsync', + messages: [apprunning([{ name: 'web', hash: 'h1' }], now - 120000), apprunning([{ name: 'probe' }], now - 60000)], + done: true, + }, + }, PEER_SOCKET); + + const [newestByIp] = messageStore.pruneAppRunningLocations.firstCall.args; + expect(newestByIp.get(SENDER)).to.deep.equal({ names: ['web'], broadcastedAt: now - 120000 }); + }); + }); + it('hands an eviction to messageStore with the time it was made', async () => { const evictStub = sinon.stub(messageStore, 'applyEviction').resolves(); const createdAt = new Date(Date.now() - 45 * 60 * 1000); @@ -2371,6 +2582,239 @@ describe('fluxCommunication tests', () => { // "nothing after this". A chunk taking its place in the queue only once its // envelope check came back put the order in the hands of whichever check // finished first. + // A response whose writes failed is not all recorded here: the peer has not + // answered, so another is asked. Credited, the node goes authoritative on a + // partial view and serves it to its own peers as complete. + describe('a response whose writes failed', () => { + const broadcast = () => ({ + version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig', + data: { ip: '1.2.3.4:16127', name: 'a', broadcastedAt: Date.now() }, + }); + + beforeEach(() => { + sinon.stub(fluxCommunicationUtils, 'verifyBroadcasts') + .callsFake(async (items) => items.map(() => ({ result: fluxCommunicationUtils.VerifyResult.OK }))); + }); + + it('is not counted as a completed appinstalling survey', async () => { + sinon.stub(messageStore, 'storeBatchAppInstallingMessages').resolves({ stored: 1, writeFailed: true }); + + await fluxCommunication.handleAppInstallingSyncResponse({ data: { type: 'fluxappinstallingsync', messages: [broadcast()], done: true } }, PEER_SOCKET); + + expect(messageStore.storeBatchAppInstallingMessages.called, 'the fixture never reached the store').to.be.true; + expect(completions).to.deep.equal([]); + }); + + it('is not counted as a completed apperrors survey', async () => { + sinon.stub(messageStore, 'storeBatchAppInstallingErrorMessages').resolves({ stored: 1, writeFailed: true }); + + await fluxCommunication.handleAppInstallingErrorsSyncResponse({ data: { type: 'fluxappinstallingerrorssync', messages: [broadcast()], done: true } }, PEER_SOCKET); + + expect(messageStore.storeBatchAppInstallingErrorMessages.called, 'the fixture never reached the store').to.be.true; + expect(completions).to.deep.equal([]); + }); + + it('is not counted as a completed apprunning survey', async () => { + sinon.stub(messageStore, 'storeBatchAppRunningMessages').resolves({ stored: 1, writeFailed: true }); + const event = { + type: 'apprunning', + envelope: { version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig' }, + data: { version: 2, ip: '1.2.3.4:16127', broadcastedAt: Date.now(), apps: [{ name: 'a' }] }, + }; + + await fluxCommunication.handleAppRunningSyncResponse({ data: { type: 'fluxapprunningsync', messages: [event], done: true } }, PEER_SOCKET); + + expect(messageStore.storeBatchAppRunningMessages.called, 'the fixture never reached the store').to.be.true; + expect(completions).to.deep.equal([]); + }); + + it('a canary: the same response written in full is counted', async () => { + sinon.stub(messageStore, 'storeBatchAppInstallingMessages').resolves({ stored: 1, writeFailed: false }); + + await fluxCommunication.handleAppInstallingSyncResponse({ data: { type: 'fluxappinstallingsync', messages: [broadcast()], done: true } }, PEER_SOCKET); + + expect(completions).to.deep.equal([{ syncType: 'appinstalling', peerKey: PEER }]); + }); + + // An answer comes as several messages and only the last says done, so the + // request ends at the message whose writes failed, wherever it falls. + describe('ends the peer\'s request at a message before its last', () => { + let writeFailures; + const onWriteFailed = (peerKey) => writeFailures.push(peerKey); + const running = () => ({ + type: 'apprunning', + envelope: { version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig' }, + data: { version: 2, ip: '1.2.3.4:16127', broadcastedAt: Date.now(), apps: [{ name: 'a' }] }, + }); + + beforeEach(() => { + writeFailures = []; + appSyncEvents.on(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + }); + + afterEach(() => { + appSyncEvents.removeListener(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + }); + + [ + ['appinstalling', 'storeBatchAppInstallingMessages', 'handleAppInstallingSyncResponse', 'fluxappinstallingsync', broadcast], + ['apperrors', 'storeBatchAppInstallingErrorMessages', 'handleAppInstallingErrorsSyncResponse', 'fluxappinstallingerrorssync', broadcast], + ['apprunning', 'storeBatchAppRunningMessages', 'handleAppRunningSyncResponse', 'fluxapprunningsync', running], + ].forEach(([syncType, store, handler, type, event]) => { + it(`for ${syncType}`, async () => { + sinon.stub(messageStore, store).resolves({ stored: 1, writeFailed: true }); + + await fluxCommunication[handler]({ data: { type, messages: [event()], done: false } }, PEER_SOCKET); + + expect(messageStore[store].called, 'the fixture never reached the store').to.be.true; + expect(writeFailures).to.deep.equal([PEER]); + }); + + it(`a canary: a ${syncType} message written in full ends nothing`, async () => { + sinon.stub(messageStore, store).resolves({ stored: 1, writeFailed: false }); + + await fluxCommunication[handler]({ data: { type, messages: [event()], done: false } }, PEER_SOCKET); + + expect(messageStore[store].called, 'the fixture never reached the store').to.be.true; + expect(writeFailures).to.deep.equal([]); + }); + }); + }); + }); + + // A call that throws part way through a message loses the rest of it, so the + // answer is not all recorded: the peer's request ends and another is asked, + // and the done that follows is not counted. + describe('a response whose calls threw', () => { + let writeFailures; + let verdict; + const onWriteFailed = (peerKey) => writeFailures.push(peerKey); + const envelope = () => ({ version: 1, timestamp: Date.now(), pubKey: '04key', signature: 'sig' }); + const broadcast = () => ({ ...envelope(), data: { ip: '1.2.3.4:16127', name: 'a', broadcastedAt: Date.now() } }); + const running = () => ({ + type: 'apprunning', envelope: envelope(), data: { version: 2, ip: '1.2.3.4:16127', broadcastedAt: Date.now(), apps: [{ name: 'a' }] }, + }); + const sigterm = () => ({ type: 'sigterm', envelope: envelope(), data: { ip: '1.2.3.4:16127', broadcastedAt: Date.now() } }); + const respond = (handler, type, messages, done) => fluxCommunication[handler]({ data: { type, messages, done } }, PEER_SOCKET); + + beforeEach(() => { + writeFailures = []; + verdict = fluxCommunicationUtils.VerifyResult.OK; + appSyncEvents.on(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + sinon.stub(fluxCommunicationUtils, 'verifyBroadcasts').callsFake(async (items) => items.map(() => ({ result: verdict }))); + sinon.stub(messageStore, 'storeBatchAppRunningMessages').resolves({ stored: 1, writeFailed: false }); + sinon.stub(messageStore, 'storeBatchAppInstallingMessages').resolves({ stored: 1, writeFailed: false }); + sinon.stub(messageStore, 'storeBatchAppInstallingErrorMessages').resolves({ stored: 1, writeFailed: false }); + sinon.stub(messageStore, 'storeAppStateEvent').resolves(); + sinon.stub(messageStore, 'expireLocationsForSigterm').resolves(); + }); + + afterEach(() => { + appSyncEvents.removeListener(SYNC_EVENTS.EPHEMERAL_SYNC_WRITE_FAILED, onWriteFailed); + }); + + [ + ['appinstalling', 'handleAppInstallingSyncResponse', 'fluxappinstallingsync', broadcast], + ['apperrors', 'handleAppInstallingErrorsSyncResponse', 'fluxappinstallingerrorssync', broadcast], + ['apprunning', 'handleAppRunningSyncResponse', 'fluxapprunningsync', running], + ].forEach(([syncType, handler, type, event]) => { + it(`ends the ${syncType} request when a broadcast it sent cannot be verified for want of the database`, async () => { + verdict = fluxCommunicationUtils.VerifyResult.UNREADABLE; + + await respond(handler, type, [event()], true); + + expect(writeFailures, 'a response the node could not verify left the request open').to.deep.equal([PEER]); + expect(completions, 'a response the node could not verify was counted').to.deep.equal([]); + }); + + it(`a canary: the same ${syncType} response verified and stored is counted`, async () => { + await respond(handler, type, [event()], true); + + expect(writeFailures).to.deep.equal([]); + expect(completions).to.deep.equal([{ syncType, peerKey: PEER }]); + }); + }); + + it('ends the appinstalling request when its store throws', async () => { + messageStore.storeBatchAppInstallingMessages.rejects(new Error('operation exceeded time limit')); + + await respond('handleAppInstallingSyncResponse', 'fluxappinstallingsync', [broadcast()], true); + + expect(writeFailures).to.deep.equal([PEER]); + expect(completions).to.deep.equal([]); + }); + + it('ends the apprunning request when pruning its locations throws', async () => { + messageStore.pruneAppRunningLocations.rejects(new Error('operation exceeded time limit')); + + await respond('handleAppRunningSyncResponse', 'fluxapprunningsync', [running()], true); + + expect(messageStore.pruneAppRunningLocations.called, 'the fixture never reached the prune').to.be.true; + expect(writeFailures).to.deep.equal([PEER]); + expect(completions).to.deep.equal([]); + }); + + it('ends the apprunning request when storing a state event throws', async () => { + messageStore.storeAppStateEvent.rejects(new Error('operation exceeded time limit')); + + await respond('handleAppRunningSyncResponse', 'fluxapprunningsync', [running(), sigterm()], true); + + expect(messageStore.storeAppStateEvent.called, 'the fixture never reached the state event').to.be.true; + expect(writeFailures).to.deep.equal([PEER]); + expect(completions).to.deep.equal([]); + }); + + it('a canary: the same apprunning response with its state event stored is counted', async () => { + await respond('handleAppRunningSyncResponse', 'fluxapprunningsync', [running(), sigterm()], true); + + expect(messageStore.storeAppStateEvent.called, 'the fixture never reached the state event').to.be.true; + expect(writeFailures).to.deep.equal([]); + expect(completions).to.deep.equal([{ syncType: 'apprunning', peerKey: PEER }]); + }); + + describe('a pending registration', () => { + const temp = () => ({ + type: 'fluxappregister', version: 1, signature: 'sig', timestamp: Date.now(), hash: 'h1', appSpecifications: { name: 'a' }, + }); + + it('this node\'s database could not store ends the apptemp request', async () => { + sinon.stub(messageStore, 'storeAppTemporaryMessage').rejects(new MongoNetworkTimeoutError('connection timed out')); + + await respond('handleTempSyncResponse', 'fluxapptempsync', [temp(), temp()], true); + + expect(writeFailures, 'a response the node could not store left the request open').to.deep.equal([PEER]); + expect(completions, 'a response the node could not store was counted').to.deep.equal([]); + sinon.assert.calledOnce(messageStore.storeAppTemporaryMessage); + }); + + it('that fails its own checks is skipped, and the rest of the answer counted', async () => { + sinon.stub(messageStore, 'storeAppTemporaryMessage') + .onFirstCall().rejects(new Error('Invalid Flux App specifications')) + .onSecondCall().resolves(true); + + await respond('handleTempSyncResponse', 'fluxapptempsync', [temp(), temp()], true); + + expect(writeFailures, 'a message the peer got wrong ended the request').to.deep.equal([]); + expect(completions).to.deep.equal([{ syncType: 'apptemp', peerKey: PEER }]); + sinon.assert.calledTwice(messageStore.storeAppTemporaryMessage); + }); + }); + + // The request this answer belonged to ended while it was being stored - the + // peer reconnected and was asked again - so a failure here ends nothing. + it('ends nothing when the request it answered has already ended', async () => { + messageStore.storeBatchAppInstallingMessages.callsFake(async () => { + wanted = () => false; + throw new Error('operation exceeded time limit'); + }); + + await respond('handleAppInstallingSyncResponse', 'fluxappinstallingsync', [broadcast()], true); + + expect(messageStore.storeBatchAppInstallingMessages.called, 'the fixture never reached the store').to.be.true; + expect(writeFailures, 'a failure ended a request it did not belong to').to.deep.equal([]); + }); + }); + describe('a chunk takes its place in the queue when it arrives', () => { const chunkFor = (marker) => ({ data: { type: 'fluxapprunningsync', messages: [{ marker }], done: false }, @@ -2835,6 +3279,18 @@ describe('fluxCommunication tests', () => { heardBack('10.0.0.1:16127'); }); + it('at startup, reads the recorded moves again on the next connection when the read fails', async () => { + registryManager.addressMovesTo.withArgs(NEW).onFirstCall().rejects(new Error('connection timed out')); + registryManager.addressMovesTo.withArgs(NEW).onSecondCall().resolves(new Map([[OLD, { to: NEW, at: Date.now() - 60000, signer: '04own' }]])); + + fluxCommunication.announceRecordedOwnMovesWhenConnected(); + + expect(await connect('10.0.0.1:16127')).to.deep.equal([]); + expect(peerManager.listeners('peerConnected').some((listener) => listener.name === 'sendOwnMovesTo')).to.equal(true); + expect(await connect('10.0.0.2:16127')).to.deep.equal([`${OLD}>${NEW}`]); + heardBack('10.0.0.2:16127'); + }); + it('at startup with nothing recorded, stops listening after the first connection', async () => { fluxCommunication.announceRecordedOwnMovesWhenConnected(); diff --git a/tests/unit/fluxNetworkHelper.test.js b/tests/unit/fluxNetworkHelper.test.js index 3ef3f02239..8cbcf2f4eb 100644 --- a/tests/unit/fluxNetworkHelper.test.js +++ b/tests/unit/fluxNetworkHelper.test.js @@ -80,7 +80,9 @@ describe('fluxNetworkHelper tests', () => { beforeEach(() => { sinon.stub(upnpService, 'isUPNP').returns(false); sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); - sinon.stub(upnpService, 'mapUpnpPort').resolves(true); + sinon.stub(upnpService, 'mapUpnpPort').resolves({ + mapped: true, code: null, reason: null, routerWide: false, + }); // The installed copy of the ufw helper is the one to run (utils/ufwHelper has its own tests). sinon.stub(ufwHelper, 'path').resolves(ufwHelper.UFW_HELPER); }); @@ -402,6 +404,68 @@ describe('fluxNetworkHelper tests', () => { }); }); + describe('waitForLocalSocketAddress tests', () => { + let benchStub; + + beforeEach(() => { + benchStub = sinon.stub(benchmarkService, 'getBenchmarks'); + fluxNetworkHelper.setLocalSocketAddress(null); + }); + + afterEach(() => { + benchStub.restore(); + fluxNetworkHelper.setLocalSocketAddress(null); + }); + + it('answers at once when benchmark reports an address', async () => { + benchStub.resolves({ status: 'success', data: { ipaddress: '203.0.113.40:16127' } }); + + expect(await fluxNetworkHelper.waitForLocalSocketAddress()).to.equal('203.0.113.40:16127'); + }); + + it('waits while no address is known, and answers with the first one learned', async () => { + benchStub.resolves({ status: 'error' }); + let answer = null; + const waiting = fluxNetworkHelper.waitForLocalSocketAddress().then((address) => { answer = address; }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(answer, 'answered before an address was known').to.equal(null); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.41:16127'); + await waiting; + + expect(answer).to.equal('203.0.113.41:16127'); + }); + + it('tells its listeners each time the node has an address after having none', () => { + const heard = []; + const listener = (address) => heard.push(address); + fluxNetworkHelper.onLocalSocketAddressKnown(listener); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.44:16127'); + fluxNetworkHelper.setLocalSocketAddress('203.0.113.44:16127'); + fluxNetworkHelper.setLocalSocketAddress(null); + fluxNetworkHelper.setLocalSocketAddress('203.0.113.44:16127'); + + expect(heard).to.deep.equal(['203.0.113.44:16127', '203.0.113.44:16127']); + }); + + it('waits again when the address it was woken for is gone before it reads it', async () => { + benchStub.resolves({ status: 'error' }); + let answer = null; + const waiting = fluxNetworkHelper.waitForLocalSocketAddress().then((address) => { answer = address; }); + await new Promise((resolve) => { setImmediate(resolve); }); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.42:16127'); + fluxNetworkHelper.setLocalSocketAddress(null); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(answer, 'answered with no address').to.equal(null); + + fluxNetworkHelper.setLocalSocketAddress('203.0.113.43:16127'); + await waiting; + expect(answer).to.equal('203.0.113.43:16127'); + }); + }); + describe('isFluxAvailable tests', () => { let stub; const ip = '127.0.0.1'; @@ -1081,6 +1145,114 @@ describe('fluxNetworkHelper tests', () => { }); }); + describe('checkMongoDbVersionAllowed tests', () => { + // minimumMongoDbAllowedVersion = '6.0.27' + const { MONGODB_FLOOR } = fluxNetworkHelper.StickyDosOwner; + + afterEach(() => { + fluxNetworkHelper.clearStickyDos(MONGODB_FLOOR); + fluxNetworkHelper.setDosStateValue(0); + fluxNetworkHelper.setDosMessage(null); + }); + + it('allows the MongoDB the fleet runs, and the floor itself', () => { + ['8.0.20', '7.0.31', '6.0.27'].forEach((version) => { + expect(fluxNetworkHelper.checkMongoDbVersionAllowed(version), version).to.equal(true); + }); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + }); + + it('takes a node below the floor out of service, and says which version it found and what to do', () => { + expect(fluxNetworkHelper.checkMongoDbVersionAllowed('6.0.26')).to.equal(false); + const reported = fluxNetworkHelper.getDOSState().data; + expect(reported.dosMessage).to.include('6.0.27'); + expect(reported.dosMessage).to.include('6.0.26'); + expect(reported.dosMessage).to.include('restart FluxOS'); + expect(reported.dosState).to.equal(100); + }); + + it('refuses every release below the floor the fleet still runs, and a pre-release of the floor', () => { + ['4.4.29', '4.4.18', '6.0.26', '6.0.27-rc0'].forEach((version) => { + fluxNetworkHelper.clearStickyDos(MONGODB_FLOOR); + expect(fluxNetworkHelper.checkMongoDbVersionAllowed(version), version).to.equal(false); + }); + }); + + it('gives no verdict when MongoDB did not report its version, or reports one that is not SemVer', () => { + [null, undefined, '', '6.0', 'dev'].forEach((version) => { + expect(fluxNetworkHelper.checkMongoDbVersionAllowed(version), String(version)).to.equal(true); + }); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + expect(fluxNetworkHelper.getDOSState().data.dosState).to.equal(0); + }); + }); + + describe('checkCgroupsAllowed tests', () => { + const { CGROUP_FLOOR } = fluxNetworkHelper.StickyDosOwner; + let root; + + // A cgroup filesystem holding the given files, with FluxOS in + // system.slice/fluxos.service. + async function cgroupRoot(files) { + root = await fs.mkdtemp(path.join(os.tmpdir(), 'cgroup-')); + const cgroupFs = path.join(root, 'cgroup'); + await Promise.all(files.map(async (file) => { + await fs.mkdir(path.dirname(path.join(cgroupFs, file)), { recursive: true }); + await fs.writeFile(path.join(cgroupFs, file), ''); + })); + const selfCgroupFile = path.join(root, 'self-cgroup'); + await fs.writeFile(selfCgroupFile, '0::/system.slice/fluxos.service\n'); + return { cgroupRoot: cgroupFs, selfCgroupFile }; + } + const everything = ['cgroup.controllers', 'system.slice/fluxos.service/cgroup.kill']; + + afterEach(async () => { + fluxNetworkHelper.clearStickyDos(CGROUP_FLOOR); + fluxNetworkHelper.setDosStateValue(0); + fluxNetworkHelper.setDosMessage(null); + await fs.rm(root, { recursive: true, force: true }); + }); + + it('allows cgroup v2 with cgroup.kill under Docker\'s systemd driver', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot(everything))).to.equal(true); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + }); + + it('takes a node without cgroup v2 out of service, and says so', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot([]))).to.equal(false); + const reported = fluxNetworkHelper.getDOSState().data; + expect(reported.dosMessage).to.include('missing: cgroup v2'); + expect(reported.dosMessage).to.include('restart FluxOS'); + expect(reported.dosState).to.equal(100); + }); + + it('asks cgroup.kill of FluxOS\'s own cgroup, not of another', async () => { + const paths = await cgroupRoot(['cgroup.controllers', 'init.scope/cgroup.kill']); + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', paths)).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup.kill (Linux 5.14 or later)'); + }); + + it('takes a node whose kernel cannot kill a cgroup out of service', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('systemd', await cgroupRoot(['cgroup.controllers']))).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup.kill (Linux 5.14 or later)'); + }); + + it('takes a node whose Docker uses another cgroup driver out of service, and names it', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed('cgroupfs', await cgroupRoot(everything))).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: Docker\'s systemd cgroup driver (found cgroupfs)'); + }); + + it('gives no verdict on a driver Docker did not report', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed(null, await cgroupRoot(everything))).to.equal(true); + expect(fluxNetworkHelper.getStickyDosMessage()).to.equal(null); + }); + + it('still refuses the kernel when Docker did not report its driver', async () => { + expect(await fluxNetworkHelper.checkCgroupsAllowed(null, await cgroupRoot([]))).to.equal(false); + expect(fluxNetworkHelper.getDOSState().data.dosMessage).to.include('missing: cgroup v2'); + }); + }); + describe('checkDockerVersionAllowed tests', () => { // minimumDockerAllowedVersion = '28.1.1' const { DOCKER_FLOOR, RESIDENTIAL_DOS } = fluxNetworkHelper.StickyDosOwner; @@ -2967,6 +3139,21 @@ describe('fluxNetworkHelper tests', () => { }); }); + it('opens and deletes a UDP test rule that names udp', async () => { + await fluxNetworkHelper.allowTestPort(31350, 'udp'); + await fluxNetworkHelper.deleteTestPortRule(31350, 'udp'); + expect(runUfw.args).to.deep.equal([ + [['allow', 'proto', 'udp', 'from', '0.0.0.0/0', 'to', 'any', 'port', '31350']], + [['delete', 'allow', 'proto', 'udp', 'from', '0.0.0.0/0', 'to', 'any', 'port', '31350']], + ]); + }); + + it('refuses a protocol other than tcp or udp, running nothing', async () => { + expect((await fluxNetworkHelper.allowTestPort(31350, 'any')).status).to.equal(false); + expect((await fluxNetworkHelper.deleteTestPortRule(31350, 'any')).status).to.equal(false); + sinon.assert.notCalled(runUfw); + }); + it('reports a held lock and refuses a port that is not a number', async () => { runUfw.resolves({ error: null, stdout: '', stderr: '', locked: true }); expect(await fluxNetworkHelper.allowTestPort(31350)).to.deep.equal({ status: false, message: 'ufw is locked by another ufw command' }); @@ -3945,6 +4132,29 @@ describe('fluxNetworkHelper tests', () => { expect(fluxNetworkHelper.isPlacementHeld()).to.equal(false); }); + it('publishes a hold when it is set, and again when its reason changes', () => { + const publish = sinon.stub(fluxEventBus, 'publish'); + + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'first'); + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'first'); + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'second'); + + expect(publish.args.filter(([name]) => name === 'placementHold:set')).to.deep.equal([ + ['placementHold:set', { owner: RESIDENTIAL_DOS, reason: 'first' }], + ['placementHold:set', { owner: RESIDENTIAL_DOS, reason: 'second' }], + ]); + }); + + it('publishes a release only of a hold that stood', () => { + fluxNetworkHelper.setPlacementHold(RESIDENTIAL_DOS, 'held'); + const publish = sinon.stub(fluxEventBus, 'publish'); + + fluxNetworkHelper.clearPlacementHold(RESIDENTIAL_DOS); + fluxNetworkHelper.clearPlacementHold(RESIDENTIAL_DOS); + + sinon.assert.calledOnceWithExactly(publish, 'placementHold:cleared', { owner: RESIDENTIAL_DOS, reason: 'held' }); + }); + it('refuses an owner it does not know, rather than minting one', () => { // An unknown owner is a caller that was never given an identity. Accepted, // it would hold the node under a name no release path knows about. diff --git a/tests/unit/fluxService.test.js b/tests/unit/fluxService.test.js index c111fe3d5b..630c5f3185 100644 --- a/tests/unit/fluxService.test.js +++ b/tests/unit/fluxService.test.js @@ -2393,7 +2393,7 @@ describe('fluxService tests', () => { }); }); - it('should return error if appsServiceAppsResources returns error', async () => { + it('should return error if appsServiceAppsResources rejects', async () => { daemonServiceControlRpcsStub.returns({ status: 'success', data: 'info data' }); daemonServiceFluxnodeRpcsStub.returns({ status: 'success', data: 'status data' }); benchmarkServiceGetInfoStub.returns({ status: 'success', data: 'info2 data' }); @@ -2401,7 +2401,7 @@ describe('fluxService tests', () => { benchmarkServiceGetBenchmarksStub.returns({ status: 'success', data: 'benchmarks data' }); appsServiceFluxUsageStub.returns({ status: 'success', data: 'usage data' }); appsServiceListRunningAppsStub.returns({ status: 'success', data: [runningContainer] }); - appsServiceAppsResourcesStub.returns({ status: 'error', data: 'appsResources data' }); + appsServiceAppsResourcesStub.rejects(new Error('appsResources data')); appsServiceGetAppHashesStub.returns({ status: 'success', data: [{ height: 694000, message: true }] }); explorerServiceStub.returns({ status: 'success', data: 'getScannedHeight data' }); fluxCommunicationStub.returns({ status: 'success', data: 'connectedPeersInfo data' }); @@ -2414,7 +2414,7 @@ describe('fluxService tests', () => { sinon.assert.calledOnceWithMatch(res.json, { status: 'error', - data: { code: undefined, name: undefined, message: 'appsResources data' }, + data: { message: 'appsResources data' }, }); }); diff --git a/tests/unit/fluxUdpTestServer.test.js b/tests/unit/fluxUdpTestServer.test.js new file mode 100644 index 0000000000..2097fcd422 --- /dev/null +++ b/tests/unit/fluxUdpTestServer.test.js @@ -0,0 +1,122 @@ +const dgram = require('node:dgram'); +const { once } = require('node:events'); +const { expect } = require('chai'); +const { + FluxUdpTestServer, UDP_PROBE_MAGIC, UDP_PROBE_BYTES, MAX_REPLIES, udpReply, +} = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); + +// The UDP half of the port test. A peer probes the port and hands back what +// replied; the requester looks for its secret in it. A UDP reply goes wherever +// the probe claims it came from, so what this server answers, and to whom, is +// the whole of its contract. +describe('FluxUdpTestServer', () => { + const TOKEN = 'c0ffee00'.repeat(4); + let server; + let port; + let client; + + const probe = (size = UDP_PROBE_BYTES, magic = UDP_PROBE_MAGIC) => { + const datagram = Buffer.alloc(size); + datagram.write(magic, 'latin1'); + return datagram; + }; + + // Sends a datagram from the client and collects what comes back in a short + // window: silence is an answer here, so the wait is the assertion's. + const exchange = async (datagram, count = 1) => { + const replies = []; + const onMessage = (message) => replies.push(message.toString()); + client.on('message', onMessage); + for (let i = 0; i < count; i += 1) { + client.send(datagram, port, '127.0.0.1'); + } + await new Promise((resolve) => { setTimeout(resolve, 150); }); + client.off('message', onMessage); + return replies; + }; + + beforeEach(async () => { + server = new FluxUdpTestServer(TOKEN); + await server.listen(0, '127.0.0.1'); + ({ port } = server.address()); + client = dgram.createSocket('udp4'); + client.bind(0, '127.0.0.1'); + await once(client, 'listening'); + }); + + afterEach(() => { + server.close(); + client.close(); + }); + + it('answers the allowed peer\'s probe with the token', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe())).to.deep.equal([udpReply(TOKEN).toString()]); + }); + + it('answers nobody until a peer is allowed', async () => { + expect(await exchange(probe())).to.deep.equal([]); + }); + + it('answers nobody but the allowed peer', async () => { + server.allow('198.51.100.7'); + + expect(await exchange(probe())).to.deep.equal([]); + }); + + it('reads an IPv4-mapped allowed address as the address it is', async () => { + server.allow('::ffff:127.0.0.1'); + + expect(await exchange(probe())).to.have.length(1); + }); + + it('never sends a reply larger than the probe it answers', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe(UDP_PROBE_BYTES - 1))).to.deep.equal([]); + expect(udpReply(TOKEN).length).to.be.at.most(UDP_PROBE_BYTES); + }); + + it('answers only a probe, not any datagram', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe(UDP_PROBE_BYTES, 'HELLO!'))).to.deep.equal([]); + }); + + it('stops answering after its reply cap', async () => { + server.allow('127.0.0.1'); + + expect(await exchange(probe(), MAX_REPLIES + 8)).to.have.length(MAX_REPLIES); + }); + + // Probes forged from one asked peer's address use up that peer's replies, + // never the next peer's. + it('gives each peer it is told to answer a fresh reply cap', async () => { + server.allow('127.0.0.1'); + expect(await exchange(probe(), MAX_REPLIES), 'canary: the cap was used up').to.have.length(MAX_REPLIES); + expect(await exchange(probe())).to.deep.equal([]); + + server.allow('127.0.0.1'); + + expect(await exchange(probe())).to.have.length(1); + }); + + it('refuses a token too long for a reply no larger than a probe', () => { + expect(() => new FluxUdpTestServer('x'.repeat(UDP_PROBE_BYTES))).to.throw(); + }); + + it('rejects listen when the port is taken', async () => { + const second = new FluxUdpTestServer(TOKEN); + let error = null; + await second.listen(port, '127.0.0.1').catch((e) => { error = e; }); + second.close(); + + expect(error && error.code).to.equal('EADDRINUSE'); + }); + + it('closes twice without throwing', () => { + server.close(); + expect(() => server.close()).to.not.throw(); + }); +}); diff --git a/tests/unit/folderStates.test.js b/tests/unit/folderStates.test.js new file mode 100644 index 0000000000..b5ef95e4f1 --- /dev/null +++ b/tests/unit/folderStates.test.js @@ -0,0 +1,158 @@ +const { expect } = require('chai'); +const folderStates = require('../../ZelBack/src/services/appMonitoring/folderStates'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); + +describe('folderStates tests', () => { + beforeEach(() => { + folderStateMap.clear(); + }); + + afterEach(() => { + folderStateMap.clear(); + }); + + describe('installed', () => { + it('records each g: and r: component of an app, and no s: or unsynced one', () => { + folderStates.installed({ + name: 'app', + version: 4, + compose: [ + { name: 'g', containerData: 'g:/data' }, + { name: 'r', containerData: 'r:/data' }, + { name: 's', containerData: 's:/data' }, + { name: 'plain', containerData: '/data' }, + ], + }); + + expect([...folderStateMap.keys()]).to.have.members(['fluxg_app', 'fluxr_app']); + expect(folderStateMap.get('fluxg_app')).to.deep.equal({ state: 'notLookedAt' }); + }); + + it('records the single folder of an app from before compose', () => { + folderStates.installed({ name: 'legacy', version: 3, containerData: 'g:/data' }); + + expect(folderStateMap.get('fluxlegacy')).to.deep.equal({ state: 'notLookedAt' }); + }); + + it('keeps the state of a folder it already has', () => { + folderStateMap.set('fluxlegacy', { state: 'heldOut' }); + + folderStates.installed({ name: 'legacy', version: 3, containerData: 'g:/data' }); + + expect(folderStateMap.get('fluxlegacy')).to.deep.equal({ state: 'heldOut' }); + }); + }); + + describe('measured', () => { + it('records what it holds as a claim', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + + expect(folderStates.claimOf('fluxa')).to.deep.equal({ bytes: 10, newestModified: 2 }); + }); + + it('records holdings it could not read as no claim', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + folderStates.measured('fluxa', null); + + expect(folderStateMap.get('fluxa')).to.deep.equal({ state: 'noClaim' }); + expect(folderStates.claimOf('fluxa')).to.equal(folderStates.NO_CLAIM_ANSWER); + }); + }); + + describe('claimOf', () => { + it('claims nothing, and is not no claim, for a folder not looked at or not held', () => { + folderStateMap.set('fluxa', { state: 'notLookedAt' }); + + expect(folderStates.claimOf('fluxa')).to.equal(undefined); + expect(folderStates.claimOf('fluxmissing')).to.equal(undefined); + }); + + it('answers no claim for a folder held out', () => { + folderStates.heldOut('fluxa'); + + expect(folderStates.claimOf('fluxa')).to.equal(folderStates.NO_CLAIM_ANSWER); + }); + }); + + describe('the folder lifecycle', () => { + it('makes a promoted folder no claim, and keeps it so once it no longer sends', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + + folderStates.sending('fluxa'); + + expect(folderStates.answer(new Set())).to.deep.equal({ + holding: { fluxa: { unreadable: true } }, notLookedAt: [], heldOut: [], volumeGone: [], + }); + }); + + it('makes a recreated volume not looked at', () => { + folderStates.measured('fluxa', { bytes: 10, newestModified: 2 }); + + folderStates.recreated('fluxa'); + + expect(folderStates.answer(new Set()).notLookedAt).to.deep.equal(['fluxa']); + }); + + it('forgets a removed folder', () => { + folderStates.heldOut('fluxa'); + + folderStates.removed('fluxa'); + + expect(folderStateMap.has('fluxa')).to.equal(false); + }); + }); + + describe('answer', () => { + it('lists each folder in one list by its state, and leaves out writable ones', () => { + folderStateMap.set('fluxwait', { state: 'notLookedAt' }); + folderStateMap.set('fluxheld', { state: 'measured', bytes: 5, newestModified: 1 }); + folderStateMap.set('fluxnone', { state: 'noClaim' }); + folderStateMap.set('fluxout', { state: 'heldOut' }); + folderStateMap.set('fluxgone', { state: 'volumeGone' }); + folderStateMap.set('fluxsend', { state: 'measured', bytes: 9, newestModified: 9 }); + folderStateMap.set('fluxsendwait', { state: 'notLookedAt' }); + + expect(folderStates.answer(new Set(['fluxsend', 'fluxsendwait']))).to.deep.equal({ + holding: { + fluxheld: { bytes: 5, newestModified: 1 }, + fluxnone: { unreadable: true }, + }, + notLookedAt: ['fluxwait'], + heldOut: ['fluxout'], + volumeGone: ['fluxgone'], + }); + }); + + it('keeps a folder whose volume image is gone when it is found unmounted without asking why, and holds out any other', () => { + folderStates.volumeGone('fluxgone'); + folderStates.measured('fluxmeasured', { bytes: 5, newestModified: 1 }); + + folderStates.notMounted('fluxgone'); + folderStates.notMounted('fluxmeasured'); + + expect(folderStateMap.get('fluxgone')).to.deep.equal({ state: 'volumeGone' }); + expect(folderStateMap.get('fluxmeasured')).to.deep.equal({ state: 'heldOut' }); + }); + + it('keeps a folder whose volume image is gone when its app\'s spec cannot be read, and holds out any other', () => { + folderStates.volumeGone('fluxgone'); + folderStates.measured('fluxmeasured', { bytes: 5, newestModified: 1 }); + folderStates.installed({ name: 'fluxwait', version: 3, containerData: 'g:/appdata' }); + + ['fluxgone', 'fluxmeasured', 'fluxwait', 'fluxnew'].forEach((folderId) => folderStates.specUnreadable(folderId)); + + expect(folderStateMap.get('fluxgone')).to.deep.equal({ state: 'volumeGone' }); + expect(folderStateMap.get('fluxmeasured')).to.deep.equal({ state: 'heldOut' }); + expect(folderStateMap.get('fluxwait')).to.deep.equal({ state: 'heldOut' }); + expect(folderStateMap.get('fluxnew')).to.deep.equal({ state: 'heldOut' }); + }); + + it('holds out a folder whose volume image was gone once the mount check finds it on the node', () => { + folderStates.volumeGone('fluxa'); + + folderStates.heldOut('fluxa'); + + expect(folderStateMap.get('fluxa')).to.deep.equal({ state: 'heldOut' }); + }); + }); +}); diff --git a/tests/unit/geolocationService.test.js b/tests/unit/geolocationService.test.js index 49fbfd3e6f..2229b3d49f 100644 --- a/tests/unit/geolocationService.test.js +++ b/tests/unit/geolocationService.test.js @@ -229,21 +229,20 @@ describe('geolocationService tests', () => { expect(geolocationService.getLastIpChangeDate()).to.equal(mockDbResult.lastIpChangeDate); }); - it('should return defaults when db connection is not available', async () => { + it('rejects when db connection is not available', async () => { dbHelperStub.databaseConnection.returns(null); - const result = await geolocationService.getNodeGeolocation(); + const error = await geolocationService.getNodeGeolocation().catch((caught) => caught); - expect(result).to.equal(null); + expect(error).to.be.an('error'); }); - it('should handle db error gracefully', async () => { + it('rejects when the stored record cannot be read', async () => { dbHelperStub.findOneInDatabase.rejects(new Error('DB error')); - const result = await geolocationService.getNodeGeolocation(); + const error = await geolocationService.getNodeGeolocation().catch((caught) => caught); - expect(result).to.equal(null); - sinon.assert.calledOnce(logStub.error); + expect(error.message).to.equal('DB error'); }); }); @@ -510,6 +509,18 @@ describe('geolocationService tests', () => { expect(geolocationService.isStaticIP()).to.equal(false); }); + it('writes nothing over a stored record it cannot read, and runs the pass again later', async () => { + fluxNetworkHelperStub.hasPublicIpOnInterface.resolves(true); + serviceHelperStub.axiosGet.resolves(ipApiResponse()); + dbHelperStub.findOneInDatabase.rejects(new Error('DB error')); + geolocationService = reload(); + + await geolocationService.setNodeGeolocation(); + + sinon.assert.notCalled(dbHelperStub.updateOneInDatabase); + expect(await geolocationService.getNodeGeolocation().catch(() => 'unreadable')).to.equal('unreadable'); + }); + it('seeds the window from the last recorded change, so an upgrade costs nothing', async () => { // ipFirstSeenAt is new, so an in-place upgrade has none. lastIpChangeDate // is persisted, restored, and says the address has been held since that @@ -1296,7 +1307,7 @@ describe('geolocationService tests', () => { await geolocationService.setNodeGeolocation(); - sinon.assert.calledWith(logStub.warn, 'Database connection not available for storing geolocation'); + sinon.assert.notCalled(dbHelperStub.updateOneInDatabase); }); it('should handle db storage error gracefully', async () => { diff --git a/tests/unit/globalconfig/default.js b/tests/unit/globalconfig/default.js index 4ca46c9505..d366672e44 100644 --- a/tests/unit/globalconfig/default.js +++ b/tests/unit/globalconfig/default.js @@ -26,9 +26,29 @@ module.exports = { fluxadm: { sshAuthorizedKeys: [], }, + // A thread of a process FluxOS depends on that stays in the kernel without + // running for one interval holds the node back from new placements, and for + // stuckIntervals intervals takes it out of service until it reboots. + hostStall: { + readIntervalMs: 5 * 60 * 1000, + stuckIntervals: 6, + stopWindowMs: 60 * 1000, + }, database: { url: database, port: 27017, + // How long one round trip, or the wait for a pooled connection, may take + // before the call fails. callTimeoutMs bounds every call on the main client: + // over four times the slowest healthy round trip seen on the fleet. + // maintenanceCallTimeoutMs bounds the client for the named long operations: + // index builds, the explorer restore's deletes and the registry rebuild. + callTimeoutMs: 60 * 1000, + maintenanceCallTimeoutMs: 15 * 60 * 1000, + // The database is suspect, and the node takes on no new apps, once + // suspectRoundTrips round trips have run out within suspectWindowMs. It stops + // being suspect suspectWindowMs after the last one. + suspectRoundTrips: 4, + suspectWindowMs: 30 * 60 * 1000, local: { database: 'zelfluxlocaltest', collections: { @@ -39,6 +59,7 @@ module.exports = { geolocation: 'geolocation', policyDocuments: 'policydocuments', // last-known-good network policy documents, so an unreachable source does not drop enforcement benchmark: 'benchmark', + nodeStartupTracker: 'nodestartuptracker', ownAddress: 'ownaddress', // the address this node last adopted as its own, and the one it moved from }, }, @@ -59,6 +80,7 @@ module.exports = { collections: { appsInformation: 'zelappsinformation', appsRuntimeState: 'zelappsruntimestate', + appsPlacements: 'zelappsplacements', }, }, appsglobal: { @@ -87,6 +109,8 @@ module.exports = { upnp: { gatewayUrl: '', nodeIp: '', + removalRetryFirstMs: 30 * 1000, + removalRetryMaxMs: 60 * 1000, }, benchmark: { host: '127.0.0.1', @@ -108,6 +132,7 @@ module.exports = { minimumNodeJsAllowedVersion: '20.8.0', minimumSyncthingAllowedVersion: '1.27.6', minimumDockerAllowedVersion: '28.1.1', + minimumMongoDbAllowedVersion: '6.0.27', fluxTeamFluxID: '1NH9BP155Rp3HSf5ef6NpUbE8JcyLRruAM', fluxSupportTeamFluxID: ['16iJqiVbHptCx87q6XQwNpKdgEZnFtKcyP'], deterministicNodesStart: 558000, @@ -241,8 +266,8 @@ module.exports = { availabilityCheckIntervalMs: 300000, availabilityRecheckMs: 60000, peerShortfallDosPerMinute: 0.13, - publicIpAskIntervalMs: 90000, - publicIpAsksBeforeDos: 8, + publicIpAskIntervalMs: 15000, + publicIpAsksBeforeDos: 48, appsPortCheck: { timeoutsMs: { default: 3600000, error: 60000, failure: 15000, dos: 300000, appError: 240000, @@ -277,8 +302,6 @@ module.exports = { daemonPONFork: 2020000, // block height where PON (Proof of Node) fork activates - chain works 4x faster after this block blocksAllowanceInterval: 1000, // ap differences can be in 1000s - more than 1 day removeBlocksAllowanceIntervalBlock: 1625000, // after this block we can start having app updates without extending subscription - block expected in April 19th 2024 - ownerAppAllowance: 1000, // in case of node owner installing some app, the app will run for this amount of blocks - temporaryAppAllowance: 200, // in case of any user installing some temporary app message for testing purposes, the app will run for this many blocks expireFluxAppsPeriod: 100, // every 100 blocks we run a check that deletes apps specifications and stops/removes the application from existence if it has been lastly updated more than 22k blocks ago updateFluxAppsPeriod: 9, // every 9 blocks we check for reinstalling of old application versions removeFluxAppsPeriod: 11, // every 11 blocks we check for more than maximum number of instances of an application @@ -292,6 +315,9 @@ module.exports = { bootDelayMultiplier: 1, spawnDelayMs: 0, removalSpacingMs: 60000, + // How long one app's removal may take, when every app is removed, before the + // next app's starts. A removal past it keeps running. + appRemovalDeadlineMs: 5 * 60 * 1000, locationTtlS: 7500, installingTtlS: 900, installErrorTtlS: 86400, @@ -367,6 +393,8 @@ module.exports = { wsHandshakeTimeoutMs: 10000, discoveryConnectionDelayMs: 500, nodeMonitorRemovalDelayMs: 60000, + // events per message of an answer to a peer's state sync + syncAnswerBatchSize: 2000, residentialCheckIntervalMs: 21600000, residentialSettleMs: 86400000, residentialEvacuationIntervalMs: 21600000, diff --git a/tests/unit/hostStallService.test.js b/tests/unit/hostStallService.test.js new file mode 100644 index 0000000000..c3018b3525 --- /dev/null +++ b/tests/unit/hostStallService.test.js @@ -0,0 +1,619 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const config = require('config'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); +const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); +const hostStallService = require('../../ZelBack/src/services/hostStallService'); + +const { stuckIntervals, readIntervalMs } = config.hostStall; + +describe('hostStallService tests', () => { + let procRoot; + let paths; + + /** + * One process in the fake /proc. + * @param {string} pid + * @param {string} comm + * @param {Array<{tid: string, state: string, switches: number, wchan?: string, startTime?: string}>} threads + * @param {{inContainer?: boolean}} [options] In a PID namespace below FluxOS's. + */ + function processWith(pid, comm, threads, { inContainer = false } = {}) { + const dir = path.join(procRoot, pid); + fs.rmSync(dir, { recursive: true, force: true }); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'comm'), `${comm}\n`); + // its id in FluxOS's namespace, then in each namespace below; no ns/ link, + // which the kernel refuses to an account that is not the process's + fs.writeFileSync(path.join(dir, 'status'), `Name:\t${comm}\nNSpid:\t${pid}${inContainer ? '\t1' : ''}\n`); + threads.forEach(({ + tid, state, switches, wchan = 'kvm_async_pf_task_wait_schedule', startTime = '1000', + }) => { + const taskDir = path.join(dir, 'task', tid); + fs.mkdirSync(taskDir, { recursive: true }); + const after = [state, ...Array(18).fill('0'), startTime, '0'].join(' '); + fs.writeFileSync(path.join(taskDir, 'stat'), `${tid} (${comm} worker) ${after}\n`); + fs.writeFileSync(path.join(taskDir, 'status'), `Name:\t${comm}\nvoluntary_ctxt_switches:\t${switches}\nnonvoluntary_ctxt_switches:\t7\n`); + fs.writeFileSync(path.join(taskDir, 'wchan'), wchan); + }); + } + + const stuckMongod = (switches = 40) => processWith('100', 'mongod', [{ tid: '101', state: 'D', switches }, { tid: '102', state: 'S', switches: 5 }]); + // The machine's uptime in the fake /proc, which a reading reads. + let uptimeMs; + function setUptime(ms) { + uptimeMs = ms; + fs.writeFileSync(path.join(procRoot, 'uptime'), `${(ms / 1000).toFixed(2)} 0.00\n`); + } + const passMs = (ms) => setUptime(uptimeMs + ms); + // A reading one interval after the last, as the service's own timer takes them. + const read = () => { + passMs(readIntervalMs); + return hostStallService.readOnce(paths); + }; + const kept = () => JSON.parse(fs.readFileSync(paths.stateFile, 'utf8')); + async function readTimes(n) { + for (let i = 0; i < n; i += 1) await read(); // eslint-disable-line no-await-in-loop + } + const placementHeld = () => fluxNetworkHelper.isPlacementHeld(); + const outOfService = () => fluxNetworkHelper.isStickyDosHeldBy(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + + beforeEach(() => { + procRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-')); + // FluxOS may follow its own ns/ link; another account's, it may not + fs.mkdirSync(path.join(procRoot, 'self', 'ns'), { recursive: true }); + fs.symlinkSync('pid:[4026531836]', path.join(procRoot, 'self', 'ns', 'pid')); + fs.writeFileSync(path.join(procRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); + paths = { procRoot, stateFile: path.join(procRoot, 'flux-host-stall.json') }; + setUptime(1000000); + }); + + afterEach(async () => { + await hostStallService.stop(); + fluxNetworkHelper.clearPlacementHold(fluxNetworkHelper.PlacementHoldOwner.HOST_STALL); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + fs.rmSync(procRoot, { recursive: true, force: true }); + sinon.restore(); + }); + + // A reading begun before a stop that ends after the next start's readings would + // replace what they have counted with what it read. + it('keeps nothing from a reading still under way when it stops', async () => { + const otherRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'hoststall-other-')); + fs.mkdirSync(path.join(otherRoot, 'self')); + fs.writeFileSync(path.join(otherRoot, 'self', 'status'), 'Name:\tnode\nNSpid:\t4242\n'); + fs.writeFileSync(path.join(otherRoot, 'uptime'), '1000.00 0.00\n'); + let finishReading = null; + const readdir = fs.promises.readdir.bind(fs.promises); + sinon.stub(fs.promises, 'readdir').callsFake((dir, ...rest) => (dir === otherRoot + ? new Promise((resolve) => { finishReading = () => resolve([]); }) + : readdir(dir, ...rest))); + stuckMongod(); + const underWay = hostStallService.readOnce({ ...paths, procRoot: otherRoot }); + while (!finishReading) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop + + hostStallService.stop(); + await readTimes(2); + finishReading(); + await underWay; + await readTimes(stuckIntervals - 1); + + expect(outOfService(), 'a reading begun before the stop reset what came after it').to.equal(true); + fs.rmSync(otherRoot, { recursive: true, force: true }); + }); + + it('holds the node back once a thread stays in D without running for one interval', async () => { + stuckMongod(); + await read(); + expect(placementHeld()).to.equal(false); + + await read(); + + expect(placementHeld()).to.equal(true); + expect(fluxNetworkHelper.getPlacementHold()).to.include('mongod thread 101').and.include('kvm_async_pf_task_wait_schedule'); + expect(outOfService()).to.equal(false); + }); + + it('takes the node out of service once the thread has not run for the configured intervals', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + expect(outOfService()).to.equal(false); + + await read(); + + expect(outOfService()).to.equal(true); + expect(fluxNetworkHelper.isNodeDos()).to.equal(true); + expect(fluxNetworkHelper.getDosMessage()).to.include('Host stall: mongod thread 101'); + }); + + it('keeps the reason it went out of service with while the thread stays stuck', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + const reason = fluxNetworkHelper.getDosMessage(); + + await readTimes(3); + + expect(fluxNetworkHelper.getDosMessage()).to.equal(reason); + }); + + it('starts the count again for a thread that ran between readings, though it is in D again', async () => { + stuckMongod(40); + await readTimes(stuckIntervals); + stuckMongod(41); + await read(); + expect(placementHeld()).to.equal(false); + + await readTimes(stuckIntervals - 1); + expect(outOfService()).to.equal(false); + + await read(); + + expect(outOfService()).to.equal(true); + }); + + it('starts the count again for a thread that left D, and releases the hold', async () => { + stuckMongod(); + await readTimes(3); + expect(placementHeld()).to.equal(true); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 40 }]); + + await read(); + + expect(placementHeld()).to.equal(false); + }); + + it('counts a reused thread id as a new thread', async () => { + stuckMongod(); + await read(); + processWith('100', 'mongod', [{ + tid: '101', state: 'D', switches: 40, startTime: '2000', + }]); + + await read(); + + expect(placementHeld()).to.equal(false); + }); + + it('reads every dependency, and no other program', async () => { + processWith('200', 'nginx', [{ tid: '201', state: 'D', switches: 3 }]); + processWith('300', 'kworker/0:1', [{ tid: '301', state: 'D', switches: 3 }]); + await readTimes(3); + expect(placementHeld()).to.equal(false); + + ['mongod', 'dockerd', 'containerd', 'syncthing', 'fluxd', 'fluxbenchd'].forEach((comm, i) => { + fs.rmSync(path.join(procRoot, '100'), { recursive: true, force: true }); + processWith('100', comm, [{ tid: `10${i}`, state: 'D', switches: 3 }]); + }); + await readTimes(2); + + expect(fluxNetworkHelper.getPlacementHold()).to.include('fluxbenchd thread 105'); + }); + + it('does not read a dependency running in an app container', async () => { + processWith('400', 'mongod', [{ tid: '401', state: 'D', switches: 3 }], { inContainer: true }); + + await readTimes(stuckIntervals + 1); + + expect(placementHeld()).to.equal(false); + expect(outOfService()).to.equal(false); + }); + + it('keeps the node out of service when the thread runs again', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 99 }]); + + await read(); + + expect(outOfService()).to.equal(true); + }); + + describe('onTrip', () => { + it('tells a listener the moment the node goes out of service, and not before', async () => { + const listener = sinon.stub(); + hostStallService.onTrip(listener); + stuckMongod(); + await readTimes(stuckIntervals); + sinon.assert.notCalled(listener); + + await read(); + + sinon.assert.calledOnce(listener); + await readTimes(2); + sinon.assert.calledOnce(listener); + }); + + it('tells a listener at once when the node is already out of service', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + const listener = sinon.stub(); + + hostStallService.onTrip(listener); + + sinon.assert.calledOnce(listener); + }); + + it('tells a listener when a kept trip is restored at start', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + const listener = sinon.stub(); + hostStallService.onTrip(listener); + + await hostStallService.start(paths); + + sinon.assert.calledOnce(listener); + }); + }); + + describe('the kept state', () => { + async function restart() { + await hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + await hostStallService.start(paths); + } + + it('keeps the trip and every thread\'s count after each reading', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + + const { trip, threads } = kept(); + expect(trip).to.equal(fluxNetworkHelper.getDosMessage()); + expect(threads.map(([, thread]) => [thread.tid, thread.intervals])).to.deep.equal([['101', stuckIntervals]]); + }); + + it('keeps no trip before the node goes out of service', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + + expect(kept().trip).to.equal(null); + }); + + it('restores the trip at a start', async () => { + stuckMongod(); + await readTimes(stuckIntervals + 1); + const reason = fluxNetworkHelper.getDosMessage(); + + await restart(); + + expect(outOfService()).to.equal(true); + expect(fluxNetworkHelper.getDosMessage()).to.equal(reason); + }); + + it('carries a stuck thread\'s count through a restart', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + await restart(); + expect(outOfService(), 'out of service before the readings reached the threshold').to.equal(false); + + await read(); + + expect(outOfService(), 'the restart started the count again').to.equal(true); + }); + + it('starts the count again for a thread that ran while FluxOS was not running', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + await restart(); + stuckMongod(41); + + await read(); + + expect(outOfService()).to.equal(false); + }); + + it('starts in service with nothing kept', async () => { + await restart(); + + expect(outOfService()).to.equal(false); + }); + + it('starts in service when the kept state cannot be read', async () => { + fs.writeFileSync(paths.stateFile, '{ not json'); + + await restart(); + + expect(outOfService()).to.equal(false); + }); + + describe('a stop while the state is being written', () => { + let release; + let reading; + beforeEach(async () => { + release = null; + const writeFile = fs.promises.writeFile.bind(fs.promises); + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.stateFile) + ? new Promise((resolve) => { release = () => resolve(writeFile(file, ...rest)); }) + : writeFile(file, ...rest))); + stuckMongod(); + reading = read(); + while (!release) await new Promise((resolve) => { setImmediate(resolve); }); // eslint-disable-line no-await-in-loop + }); + + it('still keeps the state being written', async () => { + const stopped = hostStallService.stop(); + release(); + await Promise.all([stopped, reading]); + + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + + it('resolves once the write has ended', async () => { + let done = false; + const stopped = hostStallService.stop().then(() => { done = true; }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(done, 'the stop resolved before the write ended').to.equal(false); + + release(); + await stopped; + await reading; + }); + }); + + describe('what is written', () => { + const writesOfState = () => fs.promises.writeFile.getCalls().filter((call) => String(call.args[0]).startsWith(paths.stateFile)).length; + beforeEach(() => { + sinon.spy(fs.promises, 'writeFile'); + }); + + it('writes nothing while nothing is stuck', async () => { + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 40 }]); + + await readTimes(3); + + expect(fs.existsSync(paths.stateFile)).to.equal(false); + }); + + it('writes nothing at a start with nothing kept while nothing is stuck', async () => { + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 40 }]); + + await restart(); + await readTimes(2); + + expect(fs.existsSync(paths.stateFile)).to.equal(false); + }); + + it('writes again at the next reading when a write fails', async () => { + const rename = fs.promises.rename.bind(fs.promises); + const renames = sinon.stub(fs.promises, 'rename').callsFake((from, ...rest) => rename(from, ...rest)); + renames.onFirstCall().rejects(new Error('no space left on device')); + stuckMongod(); + await read(); + + await hostStallService.readOnce(paths); + + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + + it('writes nothing for a reading that changes nothing', async () => { + stuckMongod(); + await read(); + const writes = writesOfState(); + + await hostStallService.readOnce(paths); + + expect(writesOfState()).to.equal(writes); + }); + + it('writes nothing at a start that finds what it kept unchanged, and writes once something changes', async () => { + stuckMongod(); + await readTimes(2); + const writes = writesOfState(); + await restart(); + expect(writesOfState(), 'the start wrote back what it had just read').to.equal(writes); + + await read(); + + expect(writesOfState()).to.equal(writes + 1); + }); + + it('writes again once the stuck thread runs', async () => { + stuckMongod(); + await readTimes(2); + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 41 }]); + + await read(); + + expect(kept().threads).to.deep.equal([]); + }); + }); + + describe('a write that does not end', () => { + let releases; + let holding; + const until = async (done) => { + for (let i = 0; i < 2000 && !done(); i += 1) await new Promise((resolve) => { setTimeout(resolve, 1); }); // eslint-disable-line no-await-in-loop + }; + // A write left held would hold the stop after each test, and the node out of + // service, for every test after it. + afterEach(() => { + holding = false; + releases.forEach((release) => release()); + }); + beforeEach(() => { + releases = []; + holding = true; + const writeFile = fs.promises.writeFile.bind(fs.promises); + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (holding && String(file).startsWith(paths.stateFile) + ? new Promise((resolve) => { + let released = false; + releases.push(() => { + if (released) return; + released = true; + resolve(writeFile(file, ...rest)); + }); + }) + : writeFile(file, ...rest))); + }); + + it('holds up no start: a start returns once its first reading has acted', async () => { + holding = false; + stuckMongod(); + await readTimes(stuckIntervals); + await hostStallService.stop(); + fluxNetworkHelper.clearStickyDos(fluxNetworkHelper.StickyDosOwner.HOST_STALL); + passMs(readIntervalMs); + holding = true; + + await hostStallService.start(paths); + + expect(outOfService(), 'the start\'s reading did not act').to.equal(true); + await until(() => releases.length > 0); + expect(releases.length, 'fixture: the start asked for no write').to.equal(1); + }); + + it('lands the writes one at a time and in order, the last one asked for last', async () => { + stuckMongod(); + passMs(readIntervalMs); + const first = hostStallService.readOnce(paths); + await until(() => releases.length === 1); + expect(releases.length, 'fixture: the first reading asked for no write').to.equal(1); + + processWith('100', 'mongod', [{ tid: '101', state: 'S', switches: 41 }]); + passMs(readIntervalMs); + const second = hostStallService.readOnce(paths); + await new Promise((resolve) => { setTimeout(resolve, 50); }); + expect(releases.length, 'the second write started beside the first').to.equal(1); + + releases[0](); + await until(() => releases.length === 2); + releases[1](); + await Promise.all([first, second]); + + expect(kept().threads).to.deep.equal([]); + }); + }); + + describe('a FluxOS that may not make the state directory', () => { + let stateDir; + beforeEach(() => { + stateDir = path.join(procRoot, 'run-fluxos'); + paths.stateFile = path.join(stateDir, 'host-stall.json'); + const mkdir = fs.promises.mkdir.bind(fs.promises); + sinon.stub(fs.promises, 'mkdir').callsFake((dir, ...rest) => (dir === stateDir && !fs.existsSync(stateDir) + ? Promise.reject(Object.assign(new Error('permission denied'), { code: 'EACCES' })) + : mkdir(dir, ...rest))); + }); + + const makesTheDirectory = async () => { + fs.mkdirSync(stateDir); + return { error: null, stdout: '', stderr: '' }; + }; + + it('makes it through sudo, owned by its own account, and keeps the state in it', async () => { + sinon.stub(serviceHelper, 'runCommand').callsFake(makesTheDirectory); + const { uid, gid } = os.userInfo(); + stuckMongod(); + + await read(); + + sinon.assert.calledOnceWithExactly(serviceHelper.runCommand, 'install', { + runAsRoot: true, params: ['-d', '-m', '755', '-o', String(uid), '-g', String(gid), stateDir], + }); + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + + it('tries again at the next reading when sudo fails', async () => { + const runCommand = sinon.stub(serviceHelper, 'runCommand').resolves({ error: new Error('sudo: a password is required'), stdout: '', stderr: '' }); + stuckMongod(); + await read(); + expect(fs.existsSync(paths.stateFile), 'fixture: the state was kept though sudo failed').to.equal(false); + runCommand.callsFake(makesTheDirectory); + + await hostStallService.readOnce(paths); + + expect(kept().threads.map(([, thread]) => thread.tid)).to.deep.equal(['101']); + }); + }); + + it('takes the node out of service and tells its listeners when the state cannot be kept', async () => { + const writeFile = fs.promises.writeFile.bind(fs.promises); + sinon.stub(fs.promises, 'writeFile').callsFake((file, ...rest) => (file.startsWith(paths.stateFile) + ? Promise.reject(new Error('read-only')) + : writeFile(file, ...rest))); + const listener = sinon.stub(); + hostStallService.onTrip(listener); + stuckMongod(); + + await readTimes(stuckIntervals + 1); + + expect(outOfService()).to.equal(true); + sinon.assert.calledOnce(listener); + }); + }); + + describe('the time between readings', () => { + it('adds nothing for a reading taken before an interval has passed since the count moved', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + passMs(readIntervalMs - 1000); + + await hostStallService.readOnce(paths); + + expect(outOfService()).to.equal(false); + expect(kept().threads.map(([, thread]) => thread.intervals)).to.deep.equal([stuckIntervals - 1]); + }); + + it('counts a reading one interval after the count moved, by the uptime\'s resolution', async () => { + stuckMongod(); + await readTimes(stuckIntervals); + passMs(readIntervalMs - 10); + + await hostStallService.readOnce(paths); + + expect(outOfService()).to.equal(true); + }); + + it('goes out of service only once the thread has been held for the configured intervals, however often FluxOS restarts', async () => { + const RESTART_EVERY_MS = 15000; + stuckMongod(); + await read(); + const heldSince = uptimeMs; + while (!outOfService()) { + await hostStallService.stop(); // eslint-disable-line no-await-in-loop + passMs(RESTART_EVERY_MS); + await hostStallService.start(paths); // eslint-disable-line no-await-in-loop + expect(uptimeMs - heldSince, 'never went out of service').to.be.below(2 * stuckIntervals * readIntervalMs); + } + + expect(uptimeMs - heldSince).to.be.within(stuckIntervals * readIntervalMs, stuckIntervals * readIntervalMs + RESTART_EVERY_MS); + }); + + it('takes no reading while the uptime cannot be read', async () => { + stuckMongod(); + await readTimes(2); + fs.rmSync(path.join(procRoot, 'uptime')); + stuckMongod(41); + + await hostStallService.readOnce(paths); + + expect(kept().threads.map(([, thread]) => thread.intervals)).to.deep.equal([1]); + expect(placementHeld()).to.equal(true); + }); + }); + + it('names no wait point the kernel does not show', async () => { + processWith('700', 'dockerd', [{ tid: '701', state: 'D', switches: 9, wchan: '0' }]); + + await readTimes(2); + + expect(fluxNetworkHelper.getPlacementHold()).to.include('dockerd thread 701 has been held in the kernel at unknown'); + }); + + it('reads a thread whose command name holds spaces and parentheses', async () => { + processWith('500', 'syncthing', []); + const taskDir = path.join(procRoot, '500', 'task', '501'); + fs.mkdirSync(taskDir, { recursive: true }); + fs.writeFileSync(path.join(taskDir, 'stat'), `501 (a) b (c) D ${Array(18).fill('0').join(' ')} 1000 0\n`); + fs.writeFileSync(path.join(taskDir, 'status'), 'voluntary_ctxt_switches:\t3\nnonvoluntary_ctxt_switches:\t1\n'); + fs.writeFileSync(path.join(taskDir, 'wchan'), 'folio_wait_bit'); + + await readTimes(2); + + expect(fluxNetworkHelper.getPlacementHold()).to.include('syncthing thread 501').and.include('folio_wait_bit'); + }); +}); diff --git a/tests/unit/idService.test.js b/tests/unit/idService.test.js index b32eaed203..3fc39576f7 100644 --- a/tests/unit/idService.test.js +++ b/tests/unit/idService.test.js @@ -7,6 +7,8 @@ const os = require('os'); const bitcoinMessage = require('bitcoinjs-message'); const proxyquire = require('proxyquire'); const { PassThrough } = require('stream'); +const qs = require('qs'); +const config = require('config'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); @@ -668,6 +670,7 @@ describe('idService tests', () => { loginPhrase: `${timestamp - 10000}11111111111111111111111111111`, }); sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').resolves({ loginPhrase: 'used' }); await dbHelper.initiateDB(); dbHelper.databaseConnection(); const req = { @@ -696,12 +699,44 @@ describe('idService tests', () => { sinon.assert.calledOnceWithMatch(res.json, expectedError); }); + const loginWithPhrase = async (claimed) => { + bitcoinMessageStub.returns(true); + const timestamp = Date.now(); + const message = `${timestamp - 300000}11111111111111111111111111111`; + sinon.stub(dbHelper, 'findOneInDatabase').resolves({ loginPhrase: message }); + const insertStub = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').callsFake(claimed); + await dbHelper.initiateDB(); + const mockStream = new PassThrough(); + mockStream.push(JSON.stringify({ zelid: '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg', signature: '1234356asdf', message })); + mockStream.end(); + const res = generateResponse(); + await idService.verifyLogin(mockStream, res); + await serviceHelper.delay(100); + return { res, insertStub }; + }; + + it('writes no login when its phrase cannot be marked used', async () => { + const { res, insertStub } = await loginWithPhrase(async () => { throw new Error('connection timed out'); }); + + sinon.assert.notCalled(insertStub); + sinon.assert.calledOnceWithMatch(res.json, { status: 'error' }); + }); + + it('writes no login when another request used its phrase first', async () => { + const { res, insertStub } = await loginWithPhrase(async () => null); + + sinon.assert.notCalled(insertStub); + sinon.assert.calledOnceWithMatch(res.json, { status: 'error', data: { message: 'Signed message is no longer valid. Please request a new one.' } }); + }); + it('should keep the signature off the login, in its own collection, written first and with no timer', async () => { const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); bitcoinMessageStub.returns(true); const timestamp = Date.now(); const message = `${timestamp - 300000}11111111111111111111111111111`; sinon.stub(dbHelper, 'findOneInDatabase').resolves({ loginPhrase: message }); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').resolves({ loginPhrase: message }); const insertStub = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); await dbHelper.initiateDB(); dbHelper.databaseConnection(); @@ -1253,6 +1288,339 @@ describe('idService tests', () => { }); }); + describe('a login page waiting on its phrase', () => { + const zelid = '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg'; + let findOne; + let insert; + let claim; + let phrase; + // the phrase's record: absent, or as issued, or marked used + let phraseRecord; + + const page = () => ({ send: sinon.stub(), close: sinon.stub() }); + const sentMessage = (ws) => qs.parse(ws.send.firstCall.args[0]); + async function login() { + const stream = new PassThrough(); + stream.push(JSON.stringify({ zelid, signature: '1234356asdf', message: phrase })); + stream.end(); + const res = generateResponse(); + await idService.verifyLogin(stream, res); + await serviceHelper.delay(50); + return res; + } + + beforeEach(async () => { + sinon.stub(bitcoinMessage, 'verify').returns(true); + phrase = `${Date.now() - 60000}11111111111111111111111111111`; + phraseRecord = { loginPhrase: phrase }; + findOne = sinon.stub(dbHelper, 'findOneInDatabase'); + findOne.withArgs(sinon.match.any, 'loggedusers').resolves(null); + // a login asks for the phrase unused; the page asks for it in any state + findOne.withArgs(sinon.match.any, 'activeloginphrases').callsFake(async (db, coll, query) => { + if (!phraseRecord) return null; + if (query.usedAt && phraseRecord.usedAt) return null; + return phraseRecord; + }); + insert = sinon.stub(dbHelper, 'insertOneToDatabase').resolves(true); + claim = sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').callsFake(async () => { + if (!phraseRecord || phraseRecord.usedAt) return null; + phraseRecord = { ...phraseRecord, usedAt: new Date() }; + return phraseRecord; + }); + await dbHelper.initiateDB(); + }); + + afterEach(() => { + sinon.restore(); + }); + + it('is told the login the moment it is written, and reads nothing more', async () => { + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + await login(); + + sinon.assert.calledOnce(ws.send); + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + expect(sentMessage(ws).data).to.include({ zelid, loginPhrase: phrase, message: 'Successfully logged in' }); + sinon.assert.calledOnceWithExactly(ws.close, 4012); + sinon.assert.calledOnce(findOne.withArgs(sinon.match.any, 'loggedusers')); + }); + + it('is told the login when it connects after the phrase is used and before the login is written', async () => { + let writeLogin; + insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); + const loggingIn = login(); + await serviceHelper.delay(20); + expect(phraseRecord.usedAt, 'the phrase was not used, so this proves nothing').to.be.a('date'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + writeLogin(true); + await loggingIn; + + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('is told the login when it is written while the page is still reading', async () => { + let answerRead; + findOne.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { answerRead = resolve; })); + const ws = page(); + const connecting = idService.wsRespondLoginPhrase(ws, phrase); + + await login(); + answerRead(null); + await connecting; + + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('is told the login failed once its phrase is used and the login cannot be written', async () => { + insert.withArgs(sinon.match.any, 'loggedusers').rejects(new Error('connection timed out')); + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + await login(); + + expect(sentMessage(ws)).to.deep.include({ status: 'error' }); + expect(sentMessage(ws).data.message).to.equal('connection timed out'); + }); + + it('is told its phrase is no longer valid when it expires with no login', async () => { + const clock = sinon.useFakeTimers({ now: Date.now(), toFake: ['setTimeout', 'clearTimeout', 'Date'] }); + try { + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + await clock.tickAsync(14 * 60 * 1000 - 1000); + sinon.assert.notCalled(ws.send); + + await clock.tickAsync(1000); + + expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + } finally { + clock.restore(); + } + }); + + it('is told its phrase is no longer valid when the phrase is gone and no login used it', async () => { + phraseRecord = null; + const ws = page(); + + await idService.wsRespondLoginPhrase(ws, phrase); + + expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + }); + + // The phrase is marked used before any login is written, and a login is counted + // from before it tries the phrase: a page never reads a used phrase while the + // login using it is uncounted. + it('waits, and is told the login, when it reads the phrase used while the login that used it is still being written', async () => { + let writeLogin; + insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); + const loggingIn = login(); + await serviceHelper.delay(20); + expect(phraseRecord.usedAt, 'the phrase was not used, so this proves nothing').to.be.a('date'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + writeLogin(true); + await loggingIn; + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('waits while the phrase is being marked used, and is told the login', async () => { + let markUsed; + claim.callsFake(() => new Promise((resolve) => { + markUsed = () => { + phraseRecord = { ...phraseRecord, usedAt: new Date() }; + resolve(phraseRecord); + }; + })); + const loggingIn = login(); + await serviceHelper.delay(20); + expect(markUsed, 'no login was marking the phrase, so this proves nothing').to.be.a('function'); + // the mark has landed in the database, and the login has not yet heard so + phraseRecord = { ...phraseRecord, usedAt: new Date() }; + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + markUsed(); + await loggingIn; + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('is told at once that its phrase is no longer valid when it was used by a login that has ended without writing one', async () => { + insert.withArgs(sinon.match.any, 'loggedusers').rejects(new Error('connection timed out')); + await login(); + expect(phraseRecord.usedAt, 'the phrase was not used, so this proves nothing').to.be.a('date'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + expect(sentMessage(ws).data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + }); + + it('is told the login when it reads the phrase used by a login that has written it and ended', async () => { + await login(); + findOne.withArgs(sinon.match.any, 'loggedusers').resolves({ zelid, loginPhrase: phrase }); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + // Two requests of one phrase: one uses it, the other is refused. The page waits + // on whichever is still counted, and reads again once neither is. + it('is told its phrase is no longer valid once the last login counted against it ends, when none wrote a login', async () => { + const clock = sinon.useFakeTimers({ now: Date.now(), toFake: ['setTimeout', 'clearTimeout', 'Date'], shouldAdvanceTime: true }); + try { + insert.withArgs(sinon.match.any, 'loggedusers').rejects(new Error('connection timed out')); + let refuseSecond; + claim.onSecondCall().callsFake(() => new Promise((resolve) => { refuseSecond = () => resolve(null); })); + // the second reads the phrase while it is still unused + const first = login(); + const second = login(); + await first; + expect(refuseSecond, 'the second login was not under way, so this proves nothing').to.be.a('function'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + refuseSecond(); + await second; + await clock.tickAsync(0); + + expect(sentMessage(ws).data.message, 'the page waited out the phrase instead').to.equal('Signed message is no longer valid. Please request a new one.'); + } finally { + clock.restore(); + } + }); + + it('is told the login, not that its phrase is gone, when a refused login ends while the one that used it is still writing', async () => { + let writeLogin; + insert.withArgs(sinon.match.any, 'loggedusers').callsFake(() => new Promise((resolve) => { writeLogin = resolve; })); + const first = login(); + const second = login(); + await second; + expect(writeLogin, 'the first login was not writing, so this proves nothing').to.be.a('function'); + + const ws = page(); + await idService.wsRespondLoginPhrase(ws, phrase); + sinon.assert.notCalled(ws.send); + + writeLogin(true); + await first; + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + // The login was written before the page connected, and the last login counted + // against the phrase ends while the page is still reading: the page reads again + // rather than taking the end for a phrase that signed nobody in. + it('is told the login when the last login counted ends while it is still reading', async () => { + let refuseSecond; + claim.onSecondCall().callsFake(() => new Promise((resolve) => { refuseSecond = () => resolve(null); })); + const first = login(); + const second = login(); + await first; + expect(refuseSecond, 'the second login was not under way, so this proves nothing').to.be.a('function'); + let answerRead; + const loginRow = { zelid, loginPhrase: phrase }; + findOne.withArgs(sinon.match.any, 'loggedusers').onFirstCall().callsFake(() => new Promise((resolve) => { answerRead = resolve; })); + findOne.withArgs(sinon.match.any, 'loggedusers').resolves(loginRow); + + const ws = page(); + const connecting = idService.wsRespondLoginPhrase(ws, phrase); + refuseSecond(); + await second; + answerRead(loginRow); + await connecting; + + sinon.assert.calledOnce(ws.send); + expect(sentMessage(ws)).to.deep.include({ status: 'success' }); + }); + + it('lists only the phrases no login has used as active', async () => { + const list = sinon.stub(dbHelper, 'findInDatabase').resolves([]); + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + + await idService.activeLoginPhrases({}, generateResponse()); + + expect(list.firstCall.args[2]).to.deep.equal({ usedAt: { $exists: false } }); + }); + }); + + // On the database itself: of the logins that try one phrase at once, one marks it + // used and writes its login; the rest are refused, and the phrase stays, used. + describe('a login phrase on the database', () => { + const zelid = '1Jwh4djGdRPvgLwXNGsGCoPE7uu4vihbEg'; + let phrase; + let database; + const { activeLoginPhrases, loggedUsers } = config.database.local.collections; + + async function login() { + const stream = new PassThrough(); + stream.push(JSON.stringify({ zelid, signature: '1234356asdf', message: phrase })); + stream.end(); + const res = generateResponse(); + await idService.verifyLogin(stream, res); + await serviceHelper.delay(100); + return res.json.firstCall.args[0]; + } + + beforeEach(async () => { + sinon.stub(bitcoinMessage, 'verify').returns(true); + await dbHelper.initiateDB(); + database = dbHelper.databaseConnection().db(config.database.local.database); + phrase = `${Date.now() - 60000}${Math.random().toString(36).substring(2, 15)}1111111111111111111`; + await database.collection(activeLoginPhrases).insertOne({ loginPhrase: phrase, createdAt: new Date(), expireAt: new Date(Date.now() + 900000) }); + }); + + afterEach(async () => { + sinon.restore(); + await database.collection(activeLoginPhrases).deleteMany({ loginPhrase: phrase }); + await database.collection(loggedUsers).deleteMany({ loginPhrase: phrase }); + }); + + it('signs in one of two logins that try the phrase at once, and leaves the phrase marked used', async () => { + // both have read the phrase unused before either marks it + const markUsed = dbHelper.findOneAndUpdateInDatabase; + let arrived = 0; + let bothArrived; + const barrier = new Promise((resolve) => { bothArrived = resolve; }); + sinon.stub(dbHelper, 'findOneAndUpdateInDatabase').callsFake(async (...args) => { + arrived += 1; + if (arrived === 2) bothArrived(); + await barrier; + return markUsed(...args); + }); + + const answers = await Promise.all([login(), login()]); + + expect(arrived, 'the two logins did not both try the phrase, so this proves nothing').to.equal(2); + + expect(answers.map((a) => a.status).sort()).to.deep.equal(['error', 'success']); + expect(answers.find((a) => a.status === 'error').data.message).to.equal('Signed message is no longer valid. Please request a new one.'); + expect(await database.collection(loggedUsers).countDocuments({ loginPhrase: phrase }), 'logins written').to.equal(1); + expect((await database.collection(activeLoginPhrases).findOne({ loginPhrase: phrase }))?.usedAt).to.be.a('date'); + }); + + it('refuses a login of a phrase already used', async () => { + expect((await login()).status).to.equal('success'); + + const again = await login(); + + expect(again.status).to.equal('error'); + expect(await database.collection(loggedUsers).countDocuments({ loginPhrase: phrase })).to.equal(1); + }); + }); + describe('wsRespondSignature tests', () => { const generateWebsocket = () => { const ws = {}; diff --git a/tests/unit/localChangeCheck.test.js b/tests/unit/localChangeCheck.test.js new file mode 100644 index 0000000000..fb6ef2a48b --- /dev/null +++ b/tests/unit/localChangeCheck.test.js @@ -0,0 +1,409 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); +const fluxEventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); +const syncthingFolderWrites = require('../../ZelBack/src/services/appMonitoring/syncthingFolderWrites'); + +const syncthingServiceMock = { + getDeviceId: sinon.stub(), + getDbLocalChanged: sinon.stub(), + getDbFile: sinon.stub(), + dbRevert: sinon.stub(), + getDbStatus: sinon.stub(), + getConfigFolders: sinon.stub(), +}; +const localChangeCheck = proxyquire('../../ZelBack/src/services/appMonitoring/localChangeCheck', { + '../syncthingService': syncthingServiceMock, +}); + +async function settled() { + for (let i = 0; i < 50; i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } +} + +// A read that waits until the test lets it answer. +function held(answer) { + let release; + const promise = new Promise((resolve) => { release = () => resolve(answer); }); + return { promise, release }; +} + +// The folder a revert's check was made against, as syncthing describes it. +const FOLDER_STATUS = { receiveOnlyChangedFiles: 1, receiveOnlyChangedBytes: 10 }; +const FOLDER_DEVICES = [{ deviceID: 'LOCAL-DEVICE' }, { deviceID: 'PRIMARY-DEVICE' }]; + +describe('localChangeCheck tests', () => { + beforeEach(() => { + syncthingServiceMock.getDeviceId.reset(); + syncthingServiceMock.getDeviceId.resolves('LOCAL-DEVICE'); + syncthingServiceMock.getDbLocalChanged.reset(); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [] }); + syncthingServiceMock.getDbFile.reset(); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 } }); + syncthingServiceMock.dbRevert.reset(); + syncthingServiceMock.dbRevert.resolves({ status: 'success', data: {} }); + syncthingServiceMock.getDbStatus.reset(); + syncthingServiceMock.getDbStatus.resolves(FOLDER_STATUS); + syncthingServiceMock.getConfigFolders.reset(); + syncthingServiceMock.getConfigFolders.resolves({ devices: FOLDER_DEVICES }); + }); + + // An answer with no only copy, as runCheck leaves it. + const FINGERPRINT = sinon.match(/^[0-9a-f]{64}$/); + + describe('stampOf', () => { + const COUNTS = { + receiveOnlyChangedFiles: 2, receiveOnlyChangedDirectories: 1, receiveOnlyChangedSymlinks: 0, receiveOnlyChangedDeletes: 0, receiveOnlyChangedBytes: 4096, + }; + + it('is the same for the same folder, whatever order syncthing lists its devices in', () => { + expect(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'Y' }, { deviceID: 'X' }])) + .to.equal(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'X' }, { deviceID: 'Y' }])); + }); + + // A pull moves both, and neither says anything about what this node changed. + it('does not change with this node\'s sequence or another device\'s', () => { + const devices = [{ deviceID: 'X' }]; + + expect(localChangeCheck.stampOf({ ...COUNTS, sequence: 3, remoteSequence: { A: 1 } }, devices)) + .to.equal(localChangeCheck.stampOf({ ...COUNTS, sequence: 90, remoteSequence: { A: 70 } }, devices)); + }); + + [ + ['the files this node changed', { receiveOnlyChangedFiles: 3 }], + ['the directories this node changed', { receiveOnlyChangedDirectories: 2 }], + ['the symlinks this node changed', { receiveOnlyChangedSymlinks: 1 }], + ['the deletions this node made', { receiveOnlyChangedDeletes: 1 }], + ['the bytes this node changed', { receiveOnlyChangedBytes: 8192 }], + ].forEach(([what, moved]) => { + it(`changes with ${what}`, () => { + const devices = [{ deviceID: 'X' }]; + + expect(localChangeCheck.stampOf({ ...COUNTS, ...moved }, devices)).to.not.equal(localChangeCheck.stampOf(COUNTS, devices)); + }); + }); + + it('changes with the devices the folder is shared with', () => { + expect(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'X' }])).to.not.equal(localChangeCheck.stampOf(COUNTS, [{ deviceID: 'X' }, { deviceID: 'Y' }])); + }); + }); + + describe('request', () => { + it('runs one check at a time on the node', async () => { + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a').returns(first.promise); + const a = {}; + const b = {}; + + localChangeCheck.request('fluxa_a', a, 's1', () => true); + localChangeCheck.request('fluxb_b', b, 's1', () => true); + await settled(); + + sinon.assert.neverCalledWith(syncthingServiceMock.getDbLocalChanged, 'fluxb_b'); + first.release(); + await settled(); + sinon.assert.match(a.localChangeCheck, { + stamp: 's1', onlyCopy: false, madeHere: false, changes: FINGERPRINT, + }); + sinon.assert.match(b.localChangeCheck, { + stamp: 's1', onlyCopy: false, madeHere: false, changes: FINGERPRINT, + }); + }); + + it('replaces a check still waiting with the newer one for the same folder', async () => { + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a').returns(first.promise); + const b = {}; + + localChangeCheck.request('fluxa_a', {}, 's1', () => true); + localChangeCheck.request('fluxb_b', b, 'old', () => true); + localChangeCheck.request('fluxb_b', b, 'new', () => true); + first.release(); + await settled(); + + expect(syncthingServiceMock.getDbLocalChanged.withArgs('fluxb_b').callCount, 'the folder was checked once per stamp asked for').to.equal(1); + expect(b.localChangeCheck.stamp).to.equal('new'); + }); + + it('does not start again a check already running for the same folder and stamp', async () => { + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a').returns(first.promise); + const a = {}; + + localChangeCheck.request('fluxa_a', a, 's1', () => true); + await settled(); + localChangeCheck.request('fluxa_a', a, 's1', () => true); + first.release(); + await settled(); + + sinon.assert.calledOnce(syncthingServiceMock.getDbLocalChanged); + }); + + it('says when it has answered', async () => { + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + localChangeCheck.request('fluxa_a', {}, 's1', () => true); + await settled(); + + sinon.assert.calledWithExactly(publish, 'syncthing:localChangesChecked', { folder: 'fluxa_a', onlyCopy: false, madeHere: false }); + } finally { + publish.restore(); + } + }); + }); + + describe('examineLocalChanges', () => { + it('asks about the items another device wrote, at most eight at once, and stops at the first only copy', async () => { + const files = Array.from({ length: 20 }, (_, k) => ({ name: `f${k}`, version: ['PRIMARY:1'] })); + syncthingServiceMock.getDbLocalChanged.resolves({ files }); + syncthingServiceMock.getDbFile.withArgs('fluxa_a', 'f9').resolves({ global: { localFlags: 8 } }); + + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.deep.equal({ onlyCopy: 'f9', madeHere: false }); + expect(syncthingServiceMock.getDbFile.callCount, 'asked past the batch that held it').to.equal(16); + }); + + it('says a change was made here, asking nothing about it', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); + + sinon.assert.match(await localChangeCheck.examineLocalChanges('fluxa_a'), { onlyCopy: false, madeHere: true, changes: FINGERPRINT }); + sinon.assert.notCalled(syncthingServiceMock.getDbFile); + }); + + it('says a change was made here when it is on a later page than the changes others hold', async () => { + const held = Array.from({ length: 1000 }, (_, k) => ({ name: `f${k}`, version: ['PRIMARY:1'] })); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a', 1).resolves({ files: held }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxa_a', 2).resolves({ files: [{ name: 'stray', version: ['LOCAL-D:4'] }] }); + + sinon.assert.match(await localChangeCheck.examineLocalChanges('fluxa_a'), { onlyCopy: false, madeHere: true, changes: FINGERPRINT }); + }); + + it('says nothing was made here when every change names another device', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + + sinon.assert.match(await localChangeCheck.examineLocalChanges('fluxa_a'), { onlyCopy: false, madeHere: false, changes: FINGERPRINT }); + }); + + it('cannot say when an item cannot be read', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1'] }] }); + syncthingServiceMock.getDbFile.rejects(new Error('404 No such object in the index')); + + expect(await localChangeCheck.examineLocalChanges('fluxa_a')).to.equal(null); + }); + }); + + describe('changesFingerprint', () => { + it('is the fingerprint the check left for the same changes', async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + + expect(await localChangeCheck.changesFingerprint('fluxa_a')).to.equal((await localChangeCheck.examineLocalChanges('fluxa_a')).changes); + }); + + [ + ['another name', { name: 'g', version: ['PRIMARY:1', 'LOCAL-D:2'] }], + ['another history', { name: 'f', version: ['PRIMARY:1', 'LOCAL-D:3'] }], + ['a deletion', { + name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'], deleted: true, + }], + ].forEach(([what, item]) => { + it(`differs for ${what}`, async () => { + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + const before = await localChangeCheck.changesFingerprint('fluxa_a'); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [item] }); + + expect(await localChangeCheck.changesFingerprint('fluxa_a')).to.not.equal(before); + }); + }); + + it('cannot say when the changes cannot be read', async () => { + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); + + expect(await localChangeCheck.changesFingerprint('fluxa_a')).to.equal(null); + }); + }); + + describe('revert', () => { + const CHANGED = { files: 1, directories: 0 }; + // A folder no other test has stubbed reads for, page by page. + // A folder checked with one change, its answer left on its entry. + async function checked(folderId = 'fluxr_r') { + syncthingServiceMock.getDbLocalChanged.withArgs(folderId).resolves({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + const entry = {}; + localChangeCheck.request(folderId, entry, localChangeCheck.stampOf(FOLDER_STATUS, FOLDER_DEVICES), () => true); + await settled(); + return entry; + } + + it('reverts the changes the check examined, and says so', async () => { + const entry = await checked(); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'fluxr_r'); + sinon.assert.calledWithExactly(publish, 'syncthing:localChangesReverted', { folder: 'fluxr_r', files: 1, directories: 0 }); + expect(entry).to.not.have.property('localChangeCheck'); + } finally { + publish.restore(); + } + }); + + it('does not revert changes other than the ones checked, and drops the answer', async () => { + const entry = await checked(); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').resolves({ files: [{ name: 'g', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('does not revert while the changes cannot be read, and keeps the answer', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').rejects(new Error('connect ECONNREFUSED')); + + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry.localChangeCheck).to.equal(answer); + }); + + it('does not revert a folder whose entry was replaced, or on an answer the entry no longer carries', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + + localChangeCheck.revert('fluxr_r', entry, answer, () => false, CHANGED); + await settled(); + entry.localChangeCheck = { ...answer }; + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('keeps the answer when syncthing refuses the revert, and says nothing was reverted', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + syncthingServiceMock.dbRevert.resolves({ status: 'error', data: { message: 'folder is not running' } }); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + sinon.assert.neverCalledWith(publish, 'syncthing:localChangesReverted'); + expect(entry.localChangeCheck).to.equal(answer); + } finally { + publish.restore(); + } + }); + + it('runs in the same queue as the checks, one at a time on the node', async () => { + const entry = await checked(); + const first = held({ files: [] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxb_b').returns(first.promise); + + localChangeCheck.request('fluxb_b', {}, 's1', () => true); + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + first.release(); + await settled(); + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + }); + + it('does not revert a folder a device has left since the check, and drops the answer', async () => { + const entry = await checked(); + syncthingServiceMock.getConfigFolders.resolves({ devices: [{ deviceID: 'LOCAL-DEVICE' }] }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('does not revert a folder whose counts of changes moved since the check, and drops the answer', async () => { + const entry = await checked(); + syncthingServiceMock.getDbStatus.resolves({ ...FOLDER_STATUS, receiveOnlyChangedBytes: 11 }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('does not revert while the folder cannot be read, and keeps the answer', async () => { + const entry = await checked(); + const answer = entry.localChangeCheck; + syncthingServiceMock.getConfigFolders.rejects(new Error('syncthing did not answer')); + + localChangeCheck.revert('fluxr_r', entry, answer, () => true, CHANGED); + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry.localChangeCheck).to.equal(answer); + }); + + // The order a monitor pass makes possible: the revert is asked for, and the pass's + // write of the folder's devices, which drops the primary, holds the folder first. + it('waits for a write to the folder that holds it, and does not revert once that write has dropped a device', async () => { + const entry = await checked(); + const write = held(); + const writing = syncthingFolderWrites.exclusive(['fluxr_r'], async () => { + await write.promise; + syncthingServiceMock.getConfigFolders.resolves({ devices: [{ deviceID: 'LOCAL-DEVICE' }] }); + }); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + sinon.assert.notCalled(syncthingServiceMock.getConfigFolders); + write.release(); + await writing; + await settled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(entry).to.not.have.property('localChangeCheck'); + }); + + it('holds the folder against every write until the revert is sent', async () => { + const entry = await checked(); + const sent = held({ status: 'success', data: {} }); + syncthingServiceMock.dbRevert.returns(sent.promise); + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + + let wrote = false; + const writing = syncthingFolderWrites.exclusive(['fluxr_r'], async () => { wrote = true; }); + await settled(); + expect(wrote, 'a write to the folder went in while its revert was being sent').to.equal(false); + sent.release(); + await writing; + + expect(wrote).to.equal(true); + }); + + it('does not start again a revert already running on the same answer', async () => { + const entry = await checked(); + const again = held({ files: [{ name: 'f', version: ['PRIMARY:1', 'LOCAL-D:2'] }] }); + syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').returns(again.promise); + + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + await settled(); + localChangeCheck.revert('fluxr_r', entry, entry.localChangeCheck, () => true, CHANGED); + again.release(); + await settled(); + + sinon.assert.calledOnce(syncthingServiceMock.dbRevert); + expect(syncthingServiceMock.getDbLocalChanged.withArgs('fluxr_r').callCount, 'the changes were read again for a second revert').to.equal(2); + }); + }); +}); diff --git a/tests/unit/messageStore.test.js b/tests/unit/messageStore.test.js index bec94d0cf0..d31ecf1fb4 100644 --- a/tests/unit/messageStore.test.js +++ b/tests/unit/messageStore.test.js @@ -1,9 +1,11 @@ const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); +const fluxService = require('../../ZelBack/src/services/fluxService'); describe('messageStore tests', () => { let messageStore; + let checkAndDecryptStub; let dbHelperStub; let serviceHelperStub; let messageVerifierStub; @@ -12,6 +14,7 @@ describe('messageStore tests', () => { beforeEach(() => { // Stubs + checkAndDecryptStub = sinon.stub().resolves({}); dbHelperStub = { databaseConnection: sinon.stub(), findInDatabase: sinon.stub(), @@ -82,7 +85,7 @@ describe('messageStore tests', () => { validateApplicationUpdateCompatibility: sinon.stub().resolves(), }, '../utils/enterpriseHelper': { - checkAndDecryptAppSpecs: sinon.stub().resolves({}), + checkAndDecryptAppSpecs: checkAndDecryptStub, }, '../utils/appConstants': { globalAppsMessages: 'appsMessages', @@ -213,6 +216,47 @@ describe('messageStore tests', () => { } }); + describe('enterprise registration', () => { + const enterpriseMessage = (change) => ({ + type: 'fluxappregister', + version: 1, + appSpecifications: { + name: 'entapp', version: 8, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', enterprise: 'blob', ...change, + }, + hash: 'hash123', + timestamp: Date.now(), + signature: 'sig123', + }); + + beforeEach(() => { + sinon.stub(fluxService, 'isSystemSecure').resolves(true); + messageVerifierStub.checkAppMessageExistence.resolves(null); + messageVerifierStub.checkAppTemporaryMessageExistence.resolves(null); + dbHelperStub.databaseConnection.returns({ db: sinon.stub().returns('database') }); + dbHelperStub.findOneInDatabase.resolves(null); + }); + + [ + ['an invalid app name', { name: 'ent_app' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should refuse a message with ${description} without decrypting it`, async () => { + await messageStore.storeAppTemporaryMessage(enterpriseMessage(change)).then( + () => expect.fail('should have refused'), + (error) => expect(error.message).to.equal('Invalid app name or owner'), + ); + sinon.assert.notCalled(checkAndDecryptStub); + sinon.assert.notCalled(dbHelperStub.insertOneToDatabase); + }); + }); + + it('should decrypt a message with a valid app name and owner', async () => { + // The stubs end the message after the decrypt; only reaching it matters here. + await messageStore.storeAppTemporaryMessage(enterpriseMessage({})).catch(() => {}); + sinon.assert.calledOnce(checkAndDecryptStub); + }); + }); + it('should not enforce version upgrade policy (enforced at API layer)', async () => { const message = { type: 'fluxappupdate', @@ -417,7 +461,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: false, rebroadcast: false }); + expect(result).to.deep.equal({ + stored: false, rebroadcast: false, written: [], refused: [], cleared: false, + }); expect(logStub.warn.called).to.be.true; }); @@ -438,7 +484,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: [{ name: 'testapp', hash: 'hash123' }], refused: [], cleared: false, + }); expect(dbHelperStub.updateOneInDatabase.calledOnce).to.be.true; }); @@ -461,7 +509,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: message.apps, refused: [], cleared: false, + }); expect(dbHelperStub.updateOneInDatabase.callCount).to.equal(2); // Should clean up installing records for each app (location + broadcast per app) expect(dbHelperStub.removeDocumentsFromCollection.callCount).to.equal(4); @@ -483,7 +533,9 @@ describe('messageStore tests', () => { const result = await messageStore.storeAppRunningMessage(message); - expect(result).to.deep.equal({ stored: true, rebroadcast: true }); + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: [], refused: [], cleared: true, + }); // Called three times: locations, installing locations, installing broadcasts expect(dbHelperStub.removeDocumentsFromCollection.callCount).to.equal(3); }); @@ -1099,4 +1151,65 @@ describe('messageStore tests', () => { expect($set.expireAt.getTime()).to.equal(broadcastedAt + (15 * 60 * 1000)); }); }); + + describe('a write that fails is not taken for one that landed', () => { + let failingCollection; + + beforeEach(() => { + failingCollection = { + findOne: sinon.stub().resolves({}), + updateOne: sinon.stub().rejects(new Error('connection timed out')), + bulkWrite: sinon.stub().rejects(new Error('connection timed out')), + }; + dbHelperStub.databaseConnection.returns({ db: () => ({ collection: () => failingCollection }) }); + }); + + const now = () => Date.now(); + // Every event a node records once and serves to the peers syncing from it. + const events = () => [ + ['apprunning', { signedBroadcast: { data: { ip: '1.2.3.4:16127', broadcastedAt: now(), version: 2, apps: [{ name: 'a', hash: 'h' }] } } }], + ['sigterm', { message: { ip: '1.2.3.4:16127', broadcastedAt: now() } }], + ['appremoved', { message: { ip: '1.2.3.4:16127', appName: 'a', broadcastedAt: now() } }], + ['evicted', { ip: '1.2.3.4:16127', evictedAt: now() }], + ['ipchanged', { message: { oldIP: '1.2.3.4:16127', newIP: '5.6.7.8:16127', broadcastedAt: now() } }], + ]; + + it('rejects each state event it could not record', async () => { + // eslint-disable-next-line no-restricted-syntax + for (const [type, payload] of events()) { + // eslint-disable-next-line no-await-in-loop + const outcome = await messageStore.storeAppStateEvent(type, payload).catch((err) => err); + expect(outcome, `a ${type} event was taken as recorded`).to.be.an('error'); + } + }); + + it('reports a batch of sync events it could not write', async () => { + const broadcast = { + version: 1, timestamp: now(), pubKey: 'k', signature: 's', + data: { ip: '1.2.3.4:16127', name: 'a', broadcastedAt: now() }, + }; + + const installing = await messageStore.storeBatchAppInstallingMessages([broadcast]); + const errors = await messageStore.storeBatchAppInstallingErrorMessages([broadcast]); + + expect(installing.writeFailed).to.equal(true); + expect(errors.writeFailed).to.equal(true); + }); + + it('rejects a signed claim it could not record', async () => { + dbHelperStub.updateOneInDatabase.rejects(new Error('connection timed out')); + const broadcast = { + version: 1, timestamp: now(), pubKey: 'k', signature: 's', + data: { + ip: '1.2.3.4:16127', name: 'a', hash: 'h', broadcastedAt: now(), + }, + }; + + const claim = await messageStore.storeSignedAppInstallingBroadcast(broadcast).catch((err) => err); + const error = await messageStore.storeSignedAppInstallingErrorBroadcast(broadcast).catch((err) => err); + + expect(claim).to.be.an('error'); + expect(error).to.be.an('error'); + }); + }); }); diff --git a/tests/unit/messageStoreAppRunningEntries.test.js b/tests/unit/messageStoreAppRunningEntries.test.js new file mode 100644 index 0000000000..f87ab4b6e1 --- /dev/null +++ b/tests/unit/messageStoreAppRunningEntries.test.js @@ -0,0 +1,254 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const config = require('config'); +const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const messageStore = require('../../ZelBack/src/services/appMessaging/messageStore'); +const log = require('../../ZelBack/src/lib/log'); +const { appRunningEntries } = require('../../ZelBack/src/services/utils/appRunningEntries'); +const { RUNNING_EXPIRY_MS } = require('../../ZelBack/src/services/utils/appConstants'); +const { requireMongo } = require('./dbTestHelper'); + +// A version 2 apprunning message is judged entry by entry: one entry without a +// name and hash costs its sender that entry, never the rest of the message. +describe('apprunning entries are stored one at a time', () => { + const ip = '203.0.113.21:16127'; + const otherIp = '203.0.113.22:16127'; + const weeksAgo = new Date(Date.now() - 21 * 24 * 60 * 60 * 1000); + + describe('appRunningEntries', () => { + it('keeps an entry with a string name and a string hash', () => { + const app = { name: 'web', hash: 'h1', runningSince: weeksAgo.toISOString() }; + expect(appRunningEntries([app])).to.deep.equal({ stored: [app], refused: [] }); + }); + + it('refuses each entry without a string name and hash, and only those', () => { + const good = { name: 'web', hash: 'h1' }; + const bad = [ + { name: 'probe' }, + { name: 'probe', hash: null }, + { name: 'probe', hash: 7 }, + { hash: 'h2' }, + { name: 3, hash: 'h2' }, + null, + 'web', + undefined, + ]; + const { stored, refused } = appRunningEntries([bad[0], good, ...bad.slice(1)]); + expect(stored).to.deep.equal([good]); + expect(refused).to.deep.equal(bad); + }); + + it('answers an empty list with nothing either way', () => { + expect(appRunningEntries([])).to.deep.equal({ stored: [], refused: [] }); + }); + }); + + describe('on a database', () => { + before(requireMongo); + + let locations; + + const row = (name, rowIp, broadcastedAt, runningSince = broadcastedAt) => ({ + name, + ip: rowIp, + hash: `${name}hash`, + broadcastedAt: new Date(broadcastedAt), + expireAt: new Date(broadcastedAt + RUNNING_EXPIRY_MS), + runningSince: new Date(runningSince), + }); + const message = (apps, broadcastedAt = Date.now(), from = ip) => ({ + type: 'fluxapprunning', version: 2, apps, ip: from, broadcastedAt, osUptime: 1000, staticIp: true, + }); + const rowsAt = async (rowIp = ip) => (await locations.find({ ip: rowIp }).sort({ name: 1 }).toArray()); + + beforeEach(async () => { + const database = dbHelper.databaseConnection().db(config.database.appsglobal.database); + locations = database.collection(config.database.appsglobal.collections.appsLocations); + await locations.deleteMany({ ip: { $in: [ip, otherIp] } }); + }); + + afterEach(async () => { + await locations.deleteMany({ ip: { $in: [ip, otherIp] } }); + }); + + describe('storeAppRunningMessage', () => { + it('stores the entries with a name and hash and refuses the one without', async () => { + const result = await messageStore.storeAppRunningMessage(message([ + { name: 'web', hash: 'webhash' }, + { name: 'probe' }, + { name: 'db', hash: 'dbhash' }, + ])); + + expect(result).to.deep.equal({ + stored: true, + rebroadcast: true, + written: [{ name: 'web', hash: 'webhash' }, { name: 'db', hash: 'dbhash' }], + refused: ['probe'], + cleared: false, + }); + expect((await rowsAt()).map((r) => r.name)).to.deep.equal(['db', 'web']); + }); + + // A long-running app announced beside an app installed without a hash. + it('keeps refreshing a long-running app announced beside an entry without a hash', async () => { + const lastHour = Date.now() - 60 * 60 * 1000; + await locations.insertOne(row('hytale', ip, lastHour, weeksAgo.getTime())); + + const now = Date.now(); + await messageStore.storeAppRunningMessage(message([ + { name: 'hytale', hash: 'hytalehash', runningSince: weeksAgo.toISOString() }, + { name: 'probe', runningSince: new Date(now).toISOString() }, + ], now)); + + const [hytale, ...rest] = await rowsAt(); + expect(rest).to.deep.equal([]); + expect(hytale.name).to.equal('hytale'); + expect(hytale.broadcastedAt.getTime()).to.equal(now); + expect(hytale.expireAt.getTime()).to.equal(now + RUNNING_EXPIRY_MS); + expect(hytale.runningSince.getTime()).to.equal(weeksAgo.getTime()); + }); + + it('stores nothing and relays nothing when every entry is refused', async () => { + const result = await messageStore.storeAppRunningMessage(message([{ name: 'probe' }, null])); + + expect(result).to.deep.equal({ + stored: false, rebroadcast: false, written: [], refused: ['probe', null], cleared: false, + }); + expect(await rowsAt()).to.deep.equal([]); + }); + + // An empty v2 message deletes the sender's rows. A message whose entries + // were all refused named apps; it must not be read as one that named none. + it('never reads a message whose every entry was refused as an empty snapshot', async () => { + const earlier = Date.now() - 10 * 60 * 1000; + await locations.insertOne(row('web', ip, earlier)); + + await messageStore.storeAppRunningMessage(message([{ name: 'probe' }])); + + const kept = await rowsAt(); + expect(kept.map((r) => r.name)).to.deep.equal(['web']); + expect(kept[0].broadcastedAt.getTime()).to.equal(earlier); + }); + + // Canary for the test above: the deletion it guards against is real. + it('still deletes the sender\'s older rows on a message that names no apps', async () => { + await locations.insertOne(row('web', ip, Date.now() - 10 * 60 * 1000)); + + const result = await messageStore.storeAppRunningMessage(message([])); + + expect(result).to.deep.equal({ + stored: true, rebroadcast: true, written: [], refused: [], cleared: true, + }); + expect(await rowsAt()).to.deep.equal([]); + }); + + it('reports no entry as written when a message changes no row', async () => { + const at = Date.now(); + await messageStore.storeAppRunningMessage(message([{ name: 'web', hash: 'webhash' }], at)); + + const again = await messageStore.storeAppRunningMessage(message([{ name: 'web', hash: 'webhash' }], at)); + + expect(again).to.deep.equal({ + stored: false, rebroadcast: false, written: [], refused: [], cleared: false, + }); + }); + + it('writes nothing for another node', async () => { + await locations.insertOne(row('web', otherIp, Date.now() - 10 * 60 * 1000)); + + await messageStore.storeAppRunningMessage(message([{ name: 'probe' }, { name: 'db', hash: 'dbhash' }])); + + expect((await rowsAt(otherIp)).map((r) => r.name)).to.deep.equal(['web']); + }); + + it('still refuses a version 1 message without a hash', async () => { + const result = await messageStore.storeAppRunningMessage({ + type: 'fluxapprunning', version: 1, name: 'probe', ip, broadcastedAt: Date.now(), + }); + + expect(result).to.be.instanceOf(Error); + expect(await rowsAt()).to.deep.equal([]); + }); + }); + + // The event log is what a peer syncing from this node is served. A message + // naming only entries this node refuses never takes the place of the + // sender's last announcement there. + describe('the event log', () => { + let events; + const signed = (apps, broadcastedAt = Date.now()) => ({ + version: 1, timestamp: broadcastedAt, pubKey: '04ab', signature: 'sig', data: message(apps, broadcastedAt), + }); + const storedApps = async () => (await events.find({ ip, type: 'apprunning' }).toArray()).map((event) => event.data.apps.map((a) => a.name)); + + beforeEach(async () => { + events = dbHelper.databaseConnection().db(config.database.appsglobal.database) + .collection(config.database.appsglobal.collections.appStateEvents); + await events.deleteMany({ ip }); + }); + + afterEach(async () => { + await events.deleteMany({ ip }); + }); + + it('keeps the sender\'s last announcement over a later one naming only refused entries', async () => { + await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed([{ name: 'web', hash: 'webhash' }], Date.now() - 1000) }); + await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed([{ name: 'probe' }]) }); + + expect(await storedApps()).to.deep.equal([['web']]); + }); + + it('records a mixed announcement whole, as it was signed', async () => { + await messageStore.storeAppStateEvent(messageStore.APP_STATE_EVENT_TYPES.APPRUNNING, { signedBroadcast: signed([{ name: 'web', hash: 'webhash' }, { name: 'probe' }]) }); + + expect(await storedApps()).to.deep.equal([['web', 'probe']]); + }); + + it('keeps a synced announcement naming only refused entries out of the log', async () => { + await messageStore.storeBatchAppRunningEvents([signed([{ name: 'probe' }])]); + + expect(await storedApps()).to.deep.equal([]); + }); + }); + + describe('storeBatchAppRunningMessages', () => { + const broadcast = (data) => ({ version: 1, timestamp: data.broadcastedAt, pubKey: '04ab', signature: 'sig', data }); + + it('stores the entries with a name and hash and writes no row for the one without', async () => { + const result = await messageStore.storeBatchAppRunningMessages([ + broadcast(message([{ name: 'web', hash: 'webhash' }, { name: 'probe' }, { name: 'db', hash: 'dbhash' }])), + ]); + + expect(result.writeFailed).to.equal(false); + const rows = await rowsAt(); + expect(rows.map((r) => r.name)).to.deep.equal(['db', 'web']); + expect(rows.every((r) => typeof r.hash === 'string')).to.equal(true); + }); + + it('logs the refusals of a whole batch in one line', async () => { + const warn = sinon.stub(log, 'warn'); + try { + await messageStore.storeBatchAppRunningMessages([ + broadcast(message([{ name: 'probe' }, { name: 'web', hash: 'webhash' }])), + broadcast(message([{ name: 'probe2' }, { name: 'probe3' }], Date.now(), otherIp)), + ]); + } finally { + warn.restore(); + } + + expect(warn.callCount).to.equal(1); + expect(warn.firstCall.args[0]).to.match(/^storeBatchAppRunningMessages - 3 app entries in 2 messages have no string name and hash and are not stored\./); + }); + + it('stores the other broadcasts of a batch when one has only refused entries', async () => { + await messageStore.storeBatchAppRunningMessages([ + broadcast(message([{ name: 'probe' }])), + broadcast(message([{ name: 'web', hash: 'webhash' }], Date.now(), otherIp)), + ]); + + expect(await rowsAt()).to.deep.equal([]); + expect((await rowsAt(otherIp)).map((r) => r.name)).to.deep.equal(['web']); + }); + }); + }); +}); diff --git a/tests/unit/messageVerifier.test.js b/tests/unit/messageVerifier.test.js index 21f3d05d3d..fa98513414 100644 --- a/tests/unit/messageVerifier.test.js +++ b/tests/unit/messageVerifier.test.js @@ -302,6 +302,66 @@ describe('messageVerifier tests', () => { sinon.assert.calledWith(getPreviousAppSpecsStub, sinon.match({ name: 'testapp' }), 2000000, sinon.match.number); }); + describe('a registry row that could not be written', () => { + const registration = { + type: 'fluxappregister', + version: 1, + appSpecifications: { name: 'testapp', version: 8, owner: 'owner1' }, + hash: 'hash123', + timestamp: Date.now(), + signature: 'sig123', + }; + // appPricePerMonth is stubbed at 1000 FLUX, so a registration is paid in full at that + const paidInFull = { txid: 'txid123', height: 2000000, value: 1000 * 1e8 }; + const markedResolved = () => updateOneInDatabaseStub.getCalls() + .some((call) => call.args[1] === 'appsHashes' && call.args[3]?.$set?.message === true); + + it('leaves the hash unresolved, so the promotion is completed later', async () => { + storeAppSpecificationInForceStub.rejects(new Error('connection timed out')); + const verifier = buildVerifier({ tempMessage: registration, paymentRecord: paidInFull }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(storeAppPermanentMessageStub.called, 'the fixture never reached the promotion').to.be.true; + expect(result).to.equal(false); + expect(markedResolved(), 'the hash was resolved over a registry row that was never written').to.be.false; + }); + + it('marks the hash once the row is written', async () => { + const verifier = buildVerifier({ tempMessage: registration, paymentRecord: paidInFull }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(result).to.equal(true); + expect(markedResolved()).to.be.true; + expect(storeAppSpecificationInForceStub.calledBefore(updateOneInDatabaseStub)).to.be.true; + }); + + it('writes the row of a stored message when its hash is resolved again', async () => { + const stored = { + hash: 'hash123', txid: 'txid123', height: 2000000, valueSat: 200000000, appSpecifications: { name: 'testapp' }, + }; + const verifier = buildVerifier({ + dbHelperOverrides: { + findOneInDatabase: sinon.stub() + .onFirstCall().resolves({ txid: 'txid123', height: 2000000, value: 200000000 }) + .onSecondCall().resolves(stored), // checkAppMessageExistence - held permanently + findInDatabase: sinon.stub().resolves([stored]), + }, + registryManagerOverrides: { + governingAppMessage: sinon.stub().resolves({ appSpecifications: { name: 'testapp' }, hash: 'hash123', height: 2000000 }), + }, + }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(result).to.equal(true); + sinon.assert.calledWithMatch(storeAppSpecificationInForceStub, { name: 'testapp', hash: 'hash123', height: 2000000 }); + expect(markedResolved()).to.be.true; + expect(storeAppSpecificationInForceStub.calledBefore(updateOneInDatabaseStub), 'the hash was resolved before its row was written').to.be.true; + }); + }); + describe('the payment a message is placed by', () => { it('should store the message at this node\'s record of its payment, not the caller\'s', async () => { getPreviousAppSpecsStub.resolves({ owner: 'correctOwner', version: 8 }); @@ -330,6 +390,16 @@ describe('messageVerifier tests', () => { expect(result).to.equal(false); expect(storeAppPermanentMessageStub.called).to.equal(false); }); + + it('should leave the hash unresolved when its payment record cannot be read', async () => { + const findOneInDatabase = sinon.stub().rejects(new Error('connection timed out')); + const verifier = buildVerifier({ dbHelperOverrides: { findOneInDatabase } }); + + const result = await verifier.checkAndRequestApp('hash123', 'txid123', 2000000, 200000000); + + expect(result).to.equal(false); + expect(storeAppPermanentMessageStub.called).to.equal(false); + }); }); describe('an update that has expired by itself', () => { diff --git a/tests/unit/mountParser.test.js b/tests/unit/mountParser.test.js index 05ac2206da..682bf9b4b7 100644 --- a/tests/unit/mountParser.test.js +++ b/tests/unit/mountParser.test.js @@ -440,32 +440,84 @@ describe('mountParser tests', () => { expect(result.additional[0].type).to.equal('component_primary'); }); }); - // The election decides one component per app, and an operator start defers to - // it only for that one: both read it here, so they cannot disagree. - describe('electedComponentIdentifier', () => { - it('names the first g: component of a composed app', () => { + // The election decides the app and runs every g: component on its primary; an + // operator start defers to it for the same set. Both read it here, so they + // cannot disagree. + describe('syncedComponentIdentifiers', () => { + it('names every synced component of a composed app, in compose order', () => { const spec = { name: 'App', version: 4, - compose: [{ name: 'db', containerData: '/data' }, { name: 'one', containerData: 'g:/data' }, { name: 'two', containerData: 'g:/data' }], + compose: [ + { name: 'two', containerData: 'g:/data' }, + { name: 'db', containerData: '/data' }, + { name: 'cache', containerData: 'r:/data' }, + { name: 'shared', containerData: 's:/data' }, + ], }; - expect(mountParser.electedComponentIdentifier(spec)).to.equal('one_App'); + expect(mountParser.syncedComponentIdentifiers(spec)).to.deep.equal(['two_App', 'cache_App', 'shared_App']); + }); + + it('names a v1-3 synced app by its own name', () => { + expect(mountParser.syncedComponentIdentifiers({ name: 'Old', version: 3, containerData: 'r:/data' })).to.deep.equal(['Old']); + }); + + it('names nothing for an app with no synced component, or no spec', () => { + expect(mountParser.syncedComponentIdentifiers({ name: 'Old', version: 3, containerData: '/data' })).to.deep.equal([]); + expect(mountParser.syncedComponentIdentifiers(null)).to.deep.equal([]); + }); + }); + + describe('gComponentIdentifiers', () => { + it('names every g: component of a composed app, in compose order', () => { + const spec = { + name: 'App', + version: 4, + compose: [ + { name: 'two', containerData: 'g:/data' }, + { name: 'db', containerData: '/data' }, + { name: 'one', containerData: 'g:/data' }, + { name: 'cache', containerData: 'r:/data' }, + ], + }; + expect(mountParser.gComponentIdentifiers(spec)).to.deep.equal(['two_App', 'one_App']); }); it('names a v1-3 g: app by its own name', () => { - expect(mountParser.electedComponentIdentifier({ name: 'Old', version: 3, containerData: 'g:/data' })).to.equal('Old'); + expect(mountParser.gComponentIdentifiers({ name: 'Old', version: 3, containerData: 'g:/data' })).to.deep.equal(['Old']); }); it('names nothing for an app with no g: component', () => { - expect(mountParser.electedComponentIdentifier({ name: 'Old', version: 3, containerData: 'r:/data' })).to.equal(null); - expect(mountParser.electedComponentIdentifier({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/www' }] })).to.equal(null); - expect(mountParser.electedComponentIdentifier({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/data|g:/x' }] })).to.equal(null); + expect(mountParser.gComponentIdentifiers({ name: 'Old', version: 3, containerData: 'r:/data' })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/www' }] })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/data|g:/x' }] })).to.deep.equal([]); }); it('names nothing for a spec whose components cannot be read', () => { - expect(mountParser.electedComponentIdentifier({ name: 'Ent', version: 8, enterprise: 'blob', compose: [] })).to.equal(null); - expect(mountParser.electedComponentIdentifier({ name: 'Ent', version: 8 })).to.equal(null); - expect(mountParser.electedComponentIdentifier(null)).to.equal(null); + expect(mountParser.gComponentIdentifiers({ name: 'Ent', version: 8, enterprise: 'blob', compose: [] })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers({ name: 'Ent', version: 8 })).to.deep.equal([]); + expect(mountParser.gComponentIdentifiers(null)).to.deep.equal([]); + }); + }); + + // The one g: component a release that elects a single component knows about. + describe('anchorComponentIdentifier', () => { + it('names the first g: component in compose order', () => { + const spec = { + name: 'App', + version: 4, + compose: [{ name: 'db', containerData: '/data' }, { name: 'one', containerData: 'g:/data' }, { name: 'two', containerData: 'g:/data' }], + }; + expect(mountParser.anchorComponentIdentifier(spec)).to.equal('one_App'); + }); + + it('names a v1-3 g: app by its own name', () => { + expect(mountParser.anchorComponentIdentifier({ name: 'Old', version: 3, containerData: 'g:/data' })).to.equal('Old'); + }); + + it('names nothing for an app with no g: component', () => { + expect(mountParser.anchorComponentIdentifier({ name: 'App', version: 4, compose: [{ name: 'web', containerData: '/www' }] })).to.equal(null); + expect(mountParser.anchorComponentIdentifier(null)).to.equal(null); }); }); }); diff --git a/tests/unit/moveSignerProof.test.js b/tests/unit/moveSignerProof.test.js index 59fb05e7e1..ddcc71bf51 100644 --- a/tests/unit/moveSignerProof.test.js +++ b/tests/unit/moveSignerProof.test.js @@ -10,6 +10,8 @@ const messageStore = require('../../ZelBack/src/services/appMessaging/messageSto const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); const verifyPool = require('../../ZelBack/src/services/utils/verifyPool'); const { peerManager } = require('../../ZelBack/src/services/utils/peerState'); +const { MongoNetworkTimeoutError } = require('mongodb'); +const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); const { requireMongo } = require('./dbTestHelper'); const { VerifyResult } = fluxCommunicationUtils; @@ -104,6 +106,15 @@ describe('the proof a move\'s signer spoke for the address it left', () => { expect(await fluxCommunicationUtils.verifyFluxBroadcast(await move())).to.equal(VerifyResult.NODE_NOT_FOUND); }); + it('is unreadable, not dropped, when the rows at the address it left cannot be read', async () => { + await ranAt(OLD, pubKey); + sinon.stub(dbHelper, 'databaseConnection').returns({ + db: () => ({ collection: () => ({ findOne: sinon.stub().rejects(new MongoNetworkTimeoutError('timed out')) }) }), + }); + + expect(await fluxCommunicationUtils.verifyFluxBroadcast(await move())).to.equal(VerifyResult.UNREADABLE); + }); + it('is dropped when the only apprunning stored for the address it left has expired', async () => { await ranAt(OLD, pubKey, { ago: 2 * HOUR }); @@ -263,6 +274,24 @@ describe('the proof a move\'s signer spoke for the address it left', () => { expect(await events.countDocuments({ ip: OLD, type: 'ipchanged' })).to.equal(0); }); + it('does not credit a response carrying a move whose proof cannot be read', async () => { + const completed = sinon.spy(); + appSyncEvents.on(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, completed); + try { + await sync([]); + sinon.assert.calledOnce(completed); + + sinon.stub(dbHelper, 'databaseConnection').returns({ + db: () => ({ collection: () => ({ findOne: sinon.stub().rejects(new MongoNetworkTimeoutError('timed out')) }) }), + }); + await sync([movedAt(Date.now() - 2 * MINUTE)]); + + sinon.assert.calledOnce(completed); + } finally { + appSyncEvents.off(SYNC_EVENTS.EPHEMERAL_SYNC_COMPLETE, completed); + } + }); + it('holds no rows to carry from a response sent once nothing is listed at the address the node left', async () => { await sync([apprunningAt(OLD, pubKey, Date.now() - 10 * MINUTE), movedAt(Date.now() - 2 * MINUTE)]); diff --git a/tests/unit/nodeConfirmationService.test.js b/tests/unit/nodeConfirmationService.test.js index 8f3e8cec65..8f782c3e85 100644 --- a/tests/unit/nodeConfirmationService.test.js +++ b/tests/unit/nodeConfirmationService.test.js @@ -15,6 +15,7 @@ describe('nodeConfirmationService', () => { let getFluxNodePublicKeyStub; let getFluxnodeBySocketAddressStub; let logStub; + let ownAddressKnown; beforeEach(() => { clock = sinon.useFakeTimers({ shouldAdvanceTime: false }); @@ -29,6 +30,7 @@ describe('nodeConfirmationService', () => { './fluxNetworkHelper': { getLocalSocketAddress: getLocalSocketAddressStub, getFluxNodePublicKey: getFluxNodePublicKeyStub, + onLocalSocketAddressKnown: (listener) => { ownAddressKnown = listener; }, }, './networkStateService': { getFluxnodeBySocketAddress: getFluxnodeBySocketAddressStub }, '../lib/log': logStub, @@ -155,6 +157,40 @@ describe('nodeConfirmationService', () => { }); describe('onMessageCapabilityChange', () => { + it('is heard the moment the node learns its address', async () => { + const callback = sinon.spy(); + service.onMessageCapabilityChange(callback); + setupConfirmed(); + getLocalSocketAddressStub.resolves(null); + await service.start(); + expect(service.canSendMessages()).to.be.false; + + getLocalSocketAddressStub.resolves('1.2.3.4:16127'); + ownAddressKnown('1.2.3.4:16127'); + await clock.tickAsync(0); + + expect(service.canSendMessages()).to.be.true; + sinon.assert.calledOnceWithExactly(callback, true); + }); + + it('reports one change when a poll checks while the address arriving is still being checked', async () => { + const callback = sinon.spy(); + service.onMessageCapabilityChange(callback); + setupConfirmed(); + getLocalSocketAddressStub.resolves(null); + await service.start(); + + let answerFirstCheck; + getLocalSocketAddressStub.onCall(1).returns(new Promise((resolve) => { answerFirstCheck = resolve; })); + getLocalSocketAddressStub.resolves('1.2.3.4:16127'); + ownAddressKnown('1.2.3.4:16127'); + await advancePoll(); + answerFirstCheck('1.2.3.4:16127'); + await clock.tickAsync(0); + + sinon.assert.calledOnceWithExactly(callback, true); + }); + it('should fire callback on false→true transition', async () => { const callback = sinon.spy(); service.onMessageCapabilityChange(callback); diff --git a/tests/unit/nodeHealthService.test.js b/tests/unit/nodeHealthService.test.js index 82dd97e45a..6f5d800a56 100644 --- a/tests/unit/nodeHealthService.test.js +++ b/tests/unit/nodeHealthService.test.js @@ -390,6 +390,23 @@ describe('nodeHealthService tests', () => { }); }); + // A port closed over UDP is the availability check's to act on; health + // reports it and refuses nobody for it. + it('reports UDP as unmeasured, ok or degraded, and stays fit on each', async () => { + healthyHardware(); + const readings = [[null, 'unmeasured'], [{ reachable: true }, 'ok'], [{ reachable: false, port: 31000, peers: ['a', 'b'] }, 'degraded']]; + const getUdp = sinon.stub(fluxNetworkHelper, 'getUdpReachability'); + + // eslint-disable-next-line no-restricted-syntax + for (const [reading, check] of readings) { + getUdp.returns(reading); + // eslint-disable-next-line no-await-in-loop + const fitness = await nodeHealthService.checkNodeFitness(); + expect(fitness.ok, `unfit on UDP ${check}`).to.equal(true); + expect(fitness.checks.udp).to.equal(check); + } + }); + it('checkNodeFitness returns ok with the checks when the node is fit', async () => { healthyHardware(); @@ -416,7 +433,7 @@ describe('nodeHealthService tests', () => { message: 'Node Total Ram (1) below new Cumulus requirements', failed: 'hardware', checks: { - db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'ok', hardware: 'fail', + db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'ok', udp: 'unmeasured', hardware: 'fail', }, }, }); @@ -513,7 +530,7 @@ describe('nodeHealthService tests', () => { sinon.assert.calledOnceWithExactly(res.json, { status: 'success', data: { - db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'degraded', hardware: 'unmeasured', dos: 'ok', appsDos: 'ok', + db: 'ok', syncthing: 'ok', docker: 'ok', daemon: 'degraded', udp: 'unmeasured', hardware: 'unmeasured', dos: 'ok', appsDos: 'ok', }, }); }); diff --git a/tests/unit/nodeStatusMonitor.test.js b/tests/unit/nodeStatusMonitor.test.js index e6f8f6ad04..041d28841f 100644 --- a/tests/unit/nodeStatusMonitor.test.js +++ b/tests/unit/nodeStatusMonitor.test.js @@ -24,6 +24,8 @@ describe('nodeStatusMonitor tests', () => { }, }, config: { + fluxapps: { appRemovalDeadlineMs: 300000 }, + hostStall: { stopWindowMs: 60000 }, database: { appsglobal: { database: 'globalapps', @@ -66,4 +68,440 @@ describe('nodeStatusMonitor tests', () => { // Note: This is a recursive monitoring function with complex business logic // Full testing would require integration tests or extensive mocking }); + + describe('start', () => { + it('runs no pass until boot has settled, and runs one once it has', async () => { + let settle; + const settled = new Promise((resolve) => { settle = resolve; }); + const isNodeDos = sinon.stub().returns(false); + const monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { + '../serviceHelper': { delay: () => new Promise(() => {}) }, + '../fluxNetworkHelper': { isNodeDos }, + '../nodeConfirmationService': { isDaemonStale: () => false, isConfirmed: () => false }, + '../utils/globalState': { waitForBootContainerStateSettled: () => settled }, + '../../lib/log': logStub, + }); + const installedAppsFn = sinon.stub().resolves({ status: 'success', data: [] }); + + const started = monitor.start(installedAppsFn, sinon.stub()); + await new Promise((resolve) => { setImmediate(resolve); }); + sinon.assert.notCalled(isNodeDos); + + settle(); + await started; + sinon.assert.calledOnce(isNodeDos); + }); + }); + + describe('initialize', () => { + let listeners; + let monitor; + + beforeEach(() => { + listeners = {}; + monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { + '../dbHelper': {}, + '../serviceHelper': { delay: sinon.stub().resolves() }, + '../fluxNetworkHelper': {}, + '../fluxCommunicationUtils': {}, + '../appMessaging/messageStore': {}, + '../networkStateService': {}, + '../nodeConfirmationService': { + onConfirmationChange: (cb) => { listeners.confirmation = cb; }, + onDaemonStale: (cb) => { listeners.daemonStale = cb; }, + canSendMessages: () => true, + }, + '../../lib/log': logStub, + }); + }); + + // Nothing awaits a listener's removal: a read that fails there is logged, + // never left to escape as an unhandled rejection that exits FluxOS. + it('logs a removal whose installed-apps read fails when confirmation is lost', async () => { + const installedAppsFn = sinon.stub().rejects(new Error('connection timed out')); + const removeAppLocallyFn = sinon.stub(); + monitor.initialize(installedAppsFn, removeAppLocallyFn); + + listeners.confirmation(false); + await new Promise((resolve) => { setImmediate(resolve); }); + + sinon.assert.calledOnce(installedAppsFn); + sinon.assert.notCalled(removeAppLocallyFn); + sinon.assert.calledWithMatch(logStub.error, /node lost confirmation\) failed: connection timed out/); + }); + + it('logs a removal whose installed-apps read fails when the daemon goes stale', async () => { + const installedAppsFn = sinon.stub().resolves({ status: 'error' }); + monitor.initialize(installedAppsFn, sinon.stub()); + + listeners.daemonStale(); + await new Promise((resolve) => { setImmediate(resolve); }); + + sinon.assert.calledWithMatch(logStub.error, /daemon unreachable\) failed/); + }); + }); + + describe('removal while out of service', () => { + const DEADLINE = 300000; + const STOP_WINDOW = 60000; + let clock; + let monitor; + let appKill; + let dockerService; + let onTrip; + let dos; + let bus; + const never = () => new Promise(() => {}); + const installed = (...names) => sinon.stub().resolves({ status: 'success', data: names.map((name) => ({ name })) }); + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + dos = false; + bus = { publish: sinon.stub() }; + appKill = { + readContainers: sinon.stub().resolves([]), + appsOnDisk: sinon.stub().resolves([]), + containersOf: sinon.stub().callsFake((name) => [`flux${name}`]), + stillRunnable: sinon.stub().resolves([]), + kill: sinon.stub().resolves([]), + killOutsideSlices: sinon.stub().resolves([]), + stillRunnableOutsideSlices: sinon.stub().resolves([]), + }; + dockerService = { appDockerStop: sinon.stub().resolves('stopped') }; + monitor = proxyquire('../../ZelBack/src/services/appMonitoring/nodeStatusMonitor', { + config: { + fluxapps: { appRemovalDeadlineMs: DEADLINE, nodeMonitorRemovalDelayMs: 0 }, + hostStall: { stopWindowMs: STOP_WINDOW }, + database: { appsglobal: { collections: {} } }, + }, + '../dbHelper': {}, + // the spacing between apps passes, and the wait before the next pass never ends + '../serviceHelper': { delay: sinon.stub().callsFake((ms) => (ms === 0 ? Promise.resolve() : never())) }, + '../fluxNetworkHelper': { isNodeDos: () => dos }, + '../fluxCommunicationUtils': {}, + '../appMessaging/messageStore': {}, + '../networkStateService': {}, + '../nodeConfirmationService': { + onConfirmationChange: sinon.stub(), onDaemonStale: sinon.stub(), canSendMessages: () => true, + }, + '../hostStallService': { onTrip: (listener) => { onTrip = listener; } }, + '../appLifecycle/appKill': appKill, + '../dockerService': dockerService, + '../utils/fluxEventBus': bus, + '../../lib/log': logStub, + }); + }); + + afterEach(() => { + clock.restore(); + }); + + it('publishes an app its removal removed, with nothing left to run', async () => { + await monitor.unloadForHostStall(installed('gone'), sinon.stub().resolves('removed')); + + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appUnloaded', { + app: 'gone', removal: 'removed', killed: false, runnable: [], + }); + }); + + it('publishes an app it killed, and what of it can still run, once it has stopped every app', async () => { + appKill.stillRunnable.onFirstCall().resolves(['41', '42']).onSecondCall().resolves(['42']); + + const unload = monitor.unloadForHostStall(installed('stuck'), sinon.stub().callsFake(never)); + await clock.tickAsync(STOP_WINDOW + DEADLINE); + await unload; + + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { + apps: [{ app: 'stuck', killed: true, runnable: ['42'] }], + }); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appUnloaded', { + app: 'stuck', removal: 'timedOut', killed: true, runnable: ['42'], + }); + }); + + it('asks docker to stop every container of every app at once', async () => { + appKill.containersOf.withArgs('one').returns(['fluxweb_one', 'fluxdb_one']); + dockerService.appDockerStop.callsFake(never); + + const unload = monitor.unloadForHostStall(installed('one', 'two'), sinon.stub().resolves('removed')); + await clock.tickAsync(0); + + expect(dockerService.appDockerStop.getCalls().map((call) => call.args[0])).to.have.members(['fluxweb_one', 'fluxdb_one', 'fluxtwo']); + await clock.tickAsync(STOP_WINDOW); + await unload; + }); + + it('waits one stop window for every app together, then kills what can still run', async () => { + dockerService.appDockerStop.callsFake(never); + appKill.stillRunnable.callsFake(async (name) => (appKill.kill.calledWith(name) ? [] : ['41'])); + + const unload = monitor.unloadForHostStall(installed('a', 'b', 'c'), sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW - 1); + sinon.assert.notCalled(appKill.kill); + await clock.tickAsync(1); + await unload; + + expect(appKill.kill.getCalls().map((call) => call.args[0])).to.deep.equal(['a', 'b', 'c']); + }); + + it('goes on as soon as docker has stopped every app, without waiting out the window', async () => { + const remove = sinon.stub().resolves('removed'); + + await monitor.unloadForHostStall(installed('a', 'b'), remove); + + sinon.assert.calledTwice(remove); + sinon.assert.notCalled(appKill.kill); + }); + + it('removes no app, and so tells the network of none, until every app has stopped or been killed', async () => { + dockerService.appDockerStop.callsFake(never); + appKill.stillRunnable.callsFake(async (name) => (appKill.kill.calledWith(name) ? [] : ['41'])); + const remove = sinon.stub().resolves('removed'); + + const unload = monitor.unloadForHostStall(installed('a', 'b'), remove); + await clock.tickAsync(STOP_WINDOW - 1); + sinon.assert.notCalled(remove); + await clock.tickAsync(1); + await unload; + + sinon.assert.callOrder(appKill.kill.withArgs('b'), remove.withArgs('a')); + }); + + it('removes every app once the host stalls, starting from the trip', async () => { + const remove = sinon.stub().resolves('removed'); + monitor.initialize(installed('one', 'two'), remove); + + onTrip(); + await clock.tickAsync(0); + + expect(remove.getCalls().map((call) => call.args[0])).to.deep.equal(['one', 'two']); + sinon.assert.notCalled(appKill.kill); + }); + + it('moves past a removal that does not return within the deadline', async () => { + const remove = sinon.stub(); + remove.withArgs('stuck').callsFake(never); + remove.withArgs('next').resolves('removed'); + + const unload = monitor.unloadForHostStall(installed('stuck', 'next'), remove); + await clock.tickAsync(DEADLINE - 1); + sinon.assert.neverCalledWith(remove, 'next'); + await clock.tickAsync(1); + await unload; + + sinon.assert.calledWith(remove, 'next'); + }); + + it('says an app it killed can no longer run', async () => { + appKill.stillRunnable.onFirstCall().resolves(['41']).onSecondCall().resolves([]); + + await monitor.unloadForHostStall(installed('stuck'), sinon.stub().resolves('removed')); + + sinon.assert.calledOnceWithExactly(appKill.kill, 'stuck', []); + sinon.assert.calledWithMatch(logStub.warn, /stuck can no longer run/); + }); + + it('says an app its removal could not finish can no longer run, when nothing of it is left to kill', async () => { + const remove = sinon.stub().callsFake(never); + + const unload = monitor.unloadForHostStall(installed('stopped'), remove); + await clock.tickAsync(DEADLINE); + await unload; + + sinon.assert.notCalled(appKill.kill); + sinon.assert.calledWithMatch(logStub.warn, /stopped was not removed \(timedOut\), and can no longer run/); + }); + + it('says nothing more of an app its removal removed', async () => { + await monitor.unloadForHostStall(installed('gone'), sinon.stub().resolves('removed')); + + sinon.assert.neverCalledWithMatch(logStub.warn, /can no longer run/); + }); + + it('names the processes a kill could not stop', async () => { + appKill.stillRunnable.resolves(['41', '42']); + + await monitor.unloadForHostStall(installed('app'), sinon.stub().resolves('failed')); + + sinon.assert.calledWithMatch(logStub.error, /app could not be stopped: processes 41, 42 can still run/); + }); + + it('removes the apps found on disk when the database cannot list them', async () => { + appKill.appsOnDisk.resolves(['ondisk']); + const remove = sinon.stub().resolves('removed'); + + await monitor.unloadForHostStall(sinon.stub().rejects(new Error('timed out')), remove); + + sinon.assert.calledOnceWithMatch(remove, 'ondisk'); + }); + + it('removes each app once when the database and the disk both name it', async () => { + appKill.appsOnDisk.resolves(['both']); + const remove = sinon.stub().resolves('removed'); + + await monitor.unloadForHostStall(installed('both'), remove); + + sinon.assert.calledOnce(remove); + }); + + it('reads docker\'s records once, and finds, stops, checks and kills every app from that one read', async () => { + const records = [{ id: '1', name: 'fluxa', app: 'a' }]; + appKill.readContainers.resolves(records); + appKill.stillRunnable.onFirstCall().resolves(['41']); + + await monitor.unloadForHostStall(installed('a', 'b'), sinon.stub().resolves('removed')); + + sinon.assert.calledOnce(appKill.readContainers); + sinon.assert.calledWithExactly(appKill.appsOnDisk, records); + ['containersOf', 'stillRunnable'].forEach((call) => appKill[call].getCalls().forEach(({ args }) => expect(args[1], call).to.equal(records))); + sinon.assert.calledOnceWithExactly(appKill.kill, 'a', records); + }); + + it('stops and kills every app through its slice when docker\'s records do not answer within the stop window', async () => { + appKill.readContainers.callsFake(never); + appKill.appsOnDisk.withArgs(null).resolves(['ondisk']); + dockerService.appDockerStop.callsFake(never); + appKill.stillRunnable.callsFake(async (name) => (appKill.kill.calledWith(name) ? [] : ['41'])); + + const unload = monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW * 2); + + sinon.assert.calledWithMatch(logStub.error, /container records could not be read \(no answer\)/); + expect(appKill.kill.getCalls().map(({ args }) => args)).to.deep.equal([['a', null], ['ondisk', null]]); + sinon.assert.calledWithMatch(bus.publish, 'hostStall:appsStopped'); + await unload; + }); + + it('reads the installed apps and docker\'s records at once, so a stall that holds both delays the stop by one wait', async () => { + const after = (ms, value) => () => new Promise((resolve) => { setTimeout(resolve, ms, value); }); + const installedAppsFn = sinon.stub().callsFake(after(STOP_WINDOW - 1, { status: 'success', data: [{ name: 'a' }] })); + appKill.readContainers.callsFake(after(STOP_WINDOW - 1, [])); + dockerService.appDockerStop.callsFake(never); + + const unload = monitor.unloadForHostStall(installedAppsFn, sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW - 1); + + sinon.assert.calledWith(dockerService.appDockerStop, 'fluxa'); + await clock.tickAsync(STOP_WINDOW); + await unload; + }); + + describe('when docker\'s records cannot be read', () => { + beforeEach(() => { + appKill.readContainers.resolves(null); + }); + + // A disk the database and docker share stalls both: no app is listed, and the + // containers made before the slices are found by no name. + it('kills every container outside the slices though no app is found, and says what can still run of them', async () => { + appKill.killOutsideSlices.resolves(['/sys/fs/cgroup/system.slice/docker-1.scope']); + appKill.stillRunnableOutsideSlices.resolves([]); + + const unload = monitor.unloadForHostStall(sinon.stub().callsFake(never), sinon.stub().resolves('removed')); + await clock.tickAsync(DEADLINE + STOP_WINDOW); + await unload; + + sinon.assert.calledOnce(appKill.killOutsideSlices); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { apps: [], outsideSlices: { containers: 1, runnable: [] } }); + }); + + it('kills them once whatever the number of apps, after the stop window', async () => { + dockerService.appDockerStop.callsFake(never); + + const unload = monitor.unloadForHostStall(installed('a', 'b', 'c'), sinon.stub().resolves('removed')); + await clock.tickAsync(STOP_WINDOW - 1); + sinon.assert.notCalled(appKill.killOutsideSlices); + await clock.tickAsync(1); + await unload; + + sinon.assert.calledOnce(appKill.killOutsideSlices); + }); + + it('names what of them can still run, or that it cannot be known', async () => { + appKill.stillRunnableOutsideSlices.onFirstCall().resolves(['77']); + + await monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + + sinon.assert.calledWithMatch(logStub.error, /processes 77 of containers outside the app slices can still run/); + sinon.assert.calledWithMatch(bus.publish, 'hostStall:appsStopped', { outsideSlices: { runnable: ['77'] } }); + }); + }); + + it('kills nothing outside the slices when docker\'s records name every container', async () => { + await monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + + sinon.assert.notCalled(appKill.killOutsideSlices); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { apps: [{ app: 'a', killed: false, runnable: [] }] }); + }); + + it('names a refusal to list docker\'s records', async () => { + appKill.readContainers.resolves(null); + + await monitor.unloadForHostStall(installed('a'), sinon.stub().resolves('removed')); + + sinon.assert.calledWithMatch(logStub.error, /container records could not be read \(refused\)/); + sinon.assert.calledWithExactly(appKill.appsOnDisk, null); + }); + + it('reports an app whose processes cannot all be found as unknown, never as unable to run', async () => { + appKill.stillRunnable.resolves(null); + + await monitor.unloadForHostStall(installed('app'), sinon.stub().resolves('timedOut')); + + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appsStopped', { apps: [{ app: 'app', killed: false, runnable: null }] }); + sinon.assert.calledWithExactly(bus.publish, 'hostStall:appUnloaded', { + app: 'app', removal: 'timedOut', killed: false, runnable: null, + }); + sinon.assert.calledWithMatch(logStub.error, /app: whether it can still run cannot be known/); + sinon.assert.neverCalledWithMatch(logStub.warn, /can no longer run/); + }); + + it('starts no second removal of an app whose removal is still running, and still checks it', async () => { + let finish; + const remove = sinon.stub().callsFake(() => new Promise((resolve) => { finish = resolve; })); + const first = monitor.unloadForHostStall(installed('stuck'), remove); + await clock.tickAsync(DEADLINE); + await first; + appKill.stillRunnable.resetHistory(); + + await monitor.unloadForHostStall(installed('stuck'), remove); + + sinon.assert.calledOnce(remove); + sinon.assert.calledWith(appKill.stillRunnable, 'stuck'); + finish('removed'); + await clock.tickAsync(0); + const third = monitor.unloadForHostStall(installed('stuck'), remove); + await clock.tickAsync(0); + sinon.assert.calledTwice(remove); + finish('removed'); + await third; + }); + + it('moves past a DOS removal that does not return within the deadline', async () => { + dos = true; + const remove = sinon.stub(); + remove.withArgs('stuck').callsFake(never); + remove.withArgs('next').resolves('removed'); + + monitor.monitorNodeStatus(installed('stuck', 'next'), remove); + await clock.tickAsync(DEADLINE); + + sinon.assert.calledWith(remove, 'next'); + sinon.assert.calledWithMatch(logStub.error, /stuck was not removed within 300s, moving on/); + }); + + it('starts no DOS removal while a host stall\'s removal has the node', async () => { + dos = true; + const remove = sinon.stub().callsFake(never); + const unload = monitor.unloadForHostStall(installed('held'), remove); + await clock.tickAsync(0); + + monitor.monitorNodeStatus(installed('other'), remove); + await clock.tickAsync(0); + + sinon.assert.neverCalledWith(remove, 'other'); + await clock.tickAsync(DEADLINE); + await unload; + }); + }); }); diff --git a/tests/unit/ownAddress.test.js b/tests/unit/ownAddress.test.js index 52d594f7ca..1fffd4d9e0 100644 --- a/tests/unit/ownAddress.test.js +++ b/tests/unit/ownAddress.test.js @@ -153,6 +153,29 @@ describe('fluxNetworkHelper own address tests', () => { expect(restarted.firstCall.args[0].map((app) => app.name)).to.deep.equal(['expiredrows']); }); + it('adopts nothing while whether it ran apps there cannot be read, and acts on the address again later', async () => { + const clock = sinon.useFakeTimers({ toFake: ['setTimeout'], shouldAdvanceTime: true }); + configure('198.51.100.90'); + listedHere.rejects(new Error('db down')); + + await fluxNetworkHelper.handleOwnAddress('198.51.100.91:16127'); + + sinon.assert.notCalled(moved); + sinon.assert.notCalled(fs.writeFile); + expect(await record()).to.equal(null); + + listedHere.resolves(['ranthere']); + fluxNetworkHelper.setLocalSocketAddress('198.51.100.91:16127'); + const told = new Promise((resolve) => { moved.callsFake(async () => resolve()); }); + await clock.tickAsync(60 * 1000); + await told; + // queued behind the retry, so it returns once the retry has finished + await fluxNetworkHelper.handleOwnAddress('198.51.100.91:16127'); + + sinon.assert.calledOnceWithExactly(moved, '198.51.100.90:16127', '198.51.100.91:16127', sinon.match.number); + expect(await record()).to.deep.equal({ socketAddress: '198.51.100.91:16127', movedFrom: '198.51.100.90:16127' }); + }); + it('adopts the new address without a move when nothing reports what it ran', async () => { configure('198.51.100.21'); fluxNetworkHelper.setInstalledAppsListedAt(null); @@ -578,6 +601,19 @@ describe('fluxNetworkHelper own address tests', () => { expect(await record()).to.deep.equal({ socketAddress: '198.51.100.9:16157', movedFrom: '198.51.100.9:16127' }); }); + it('adopts nothing from a listing at its own configured IP that nothing answers, while whether it ran apps there cannot be read', async () => { + configure('198.51.100.92'); + collateralListedAt('198.51.100.92:16127', '198.51.100.93:16127'); + listedHere.rejects(new Error('db down')); + + await fluxNetworkHelper.checkDeterministicNodesCollisions(); + + sinon.assert.notCalled(moved); + sinon.assert.notCalled(fs.writeFile); + sinon.assert.notCalled(daemonServiceFluxnodeRpcs.createConfirmationTransaction); + expect(await record()).to.equal(null); + }); + it('adopts the new address without a move from a listing at its own configured IP that nothing answers, where it ran no apps', async () => { configure('198.51.100.25'); collateralListedAt('198.51.100.25:16127', '198.51.100.26:16127'); diff --git a/tests/unit/paymentPositions.test.js b/tests/unit/paymentPositions.test.js index 809c7a8248..8db1626197 100644 --- a/tests/unit/paymentPositions.test.js +++ b/tests/unit/paymentPositions.test.js @@ -11,7 +11,8 @@ const dbHelper = require('../../ZelBack/src/services/dbHelper'); const daemonServiceUtils = require('../../ZelBack/src/services/daemonService/daemonServiceUtils'); const messageVerifier = require('../../ZelBack/src/services/appMessaging/messageVerifier'); const { appPaymentAddresses, appPaymentPositions } = require('../../ZelBack/src/services/appPaymentPositions'); -const { backfillPaymentPositions } = require('../../ZelBack/src/services/migrations/paymentPositions'); +const { backfillPaymentPositions, BACKFILL_BATCH } = require('../../ZelBack/src/services/migrations/paymentPositions'); +const { requireMongo } = require('./dbTestHelper'); chai.use(chaiAsPromised); const { expect } = chai; @@ -85,6 +86,41 @@ describe('app payment positions', () => { }); }); + describe('backfillPaymentPositions against the database', () => { + before(requireMongo); + + let hashes; + + beforeEach(async () => { + await dbHelper.initiateDB(); + hashes = dbHelper.databaseConnection().db(config.database.daemon.database).collection(config.database.daemon.collections.appsHashes); + await hashes.deleteMany({}); + sinon.stub(messageVerifier, 'moveStoredMessagesToPayments').resolves([]); + sinon.stub(messageVerifier, 'storeGoverningAppSpecifications').resolves(); + executeCall.withArgs('getBlockCount').resolves({ status: 'success', data: 1000 }); + executeCall.withArgs('getaddressdeltas').resolves({ status: 'success', data: [] }); + }); + + afterEach(async () => { + await hashes.deleteMany({}); + }); + + it('asks the daemon nothing for a record it already found not on the chain', async () => { + await hashes.insertOne({ + hash: 'orphaned', txid: 'orphaned', height: 500, value: 1, + }); + + expect((await backfillPaymentPositions()).notOnChain).to.equal(1); + expect((await hashes.findOne({ hash: 'orphaned' })).notOnChain).to.equal(true); + executeCall.resetHistory(); + + expect(await backfillPaymentPositions()).to.deep.equal({ + positioned: 0, heightsCorrected: 0, notOnChain: 0, aboveTip: 0, + }); + sinon.assert.notCalled(executeCall); + }); + }); + describe('backfillPaymentPositions', () => { let findInDatabase; let bulkWriteInDatabase; @@ -117,7 +153,7 @@ describe('app payment positions', () => { findInDatabase.resolves([]); await backfillPaymentPositions(); expect(findInDatabase.firstCall.args[1]).to.equal(config.database.daemon.collections.appsHashes); - expect(findInDatabase.firstCall.args[2]).to.deep.equal({ txIndex: { $exists: false } }); + expect(findInDatabase.firstCall.args[2]).to.deep.equal({ txIndex: { $exists: false }, notOnChain: { $ne: true } }); }); it('positions a record, at the height the chain holds its transaction', async () => { @@ -213,6 +249,18 @@ describe('app payment positions', () => { }); }); + it('writes the records in batches, each of at most BACKFILL_BATCH', async () => { + const count = (2 * BACKFILL_BATCH) + 1; + const records = Array.from({ length: count }, (_, i) => ({ txid: `t${i}`, height: 500 })); + findInDatabase.resolves(records); + executeCall.withArgs('getaddressdeltas').resolves({ status: 'success', data: records.map(({ txid }, i) => delta(txid, 500, i)) }); + + const outcome = await backfillPaymentPositions(); + + expect(bulkWriteInDatabase.getCalls().map((call) => call.args[2].length)).to.deep.equal([BACKFILL_BATCH, BACKFILL_BATCH, 1]); + expect(outcome.positioned).to.equal(count); + }); + it('writes only to a record that still has no position', async () => { findInDatabase.resolves([{ txid: 'a', height: 500 }]); executeCall.withArgs('getaddressdeltas').resolves({ status: 'success', data: [delta('a', 500, 0)] }); diff --git a/tests/unit/peerComponent.test.js b/tests/unit/peerComponent.test.js index 0d760df30c..c2abcdd6bd 100644 --- a/tests/unit/peerComponent.test.js +++ b/tests/unit/peerComponent.test.js @@ -5,12 +5,17 @@ const { expect } = require('chai'); const sinon = require('sinon'); const axios = require('axios'); const peerIdentityService = require('../../ZelBack/src/services/peerIdentityService'); -const { PeerComponent, componentStateOnPeers, componentStateOnOtherHolders } = require('../../ZelBack/src/services/appMonitoring/peerComponent'); +const syncthingService = require('../../ZelBack/src/services/syncthingService'); +const registryManager = require('../../ZelBack/src/services/appDatabase/registryManager'); +const globalState = require('../../ZelBack/src/services/utils/globalState'); +const { + PeerComponent, componentStateOnPeers, componentStateOnOtherHolders, peerComponentState, peerUndecided, +} = require('../../ZelBack/src/services/appMonitoring/peerComponent'); describe('peerComponent tests', () => { const appId = 'fluxw_a'; const ctx = { - appId, identifier: 'w_a', appName: 'a', liveness: {}, logPrefix: 'test', + appIds: [appId], identifier: 'w_a', appName: 'a', liveness: {}, logPrefix: 'test', }; // Each peer answers /apps/heldcomponents with what it holds; a peer given a @@ -105,4 +110,94 @@ describe('peerComponent tests', () => { expect(state).to.equal(PeerComponent.UNKNOWN); }); }); + + // An app with more than one g: component is held by the node that holds any of + // them: its primary runs them together. + describe('an app with more than one g: component', () => { + const appIds = ['fluxdb_a', 'fluxweb_a']; + const appCtx = { + appIds, identifier: 'db_a', appName: 'a', logPrefix: 'test', label: 'peer', + liveness: { localConnectivity: () => ({ connected: true }) }, + }; + const PEER = '2.2.2.2:16127'; + + it('reads a peer holding only a later g: component as holding the app', async () => { + peersAnswer({ [PEER]: { held: ['fluxweb_a'] } }); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.RUNNING); + }); + + it('reads a peer holding none of them as not holding the app', async () => { + peersAnswer({ [PEER]: { held: ['fluxweb_b', 'fluxother_a'] } }); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); + }); + + // A whole-name match, as for one component: another app whose component + // names merely contain these must not read as this app. + it('does not read a longer container name as one of the app\'s', async () => { + peersAnswer({ [PEER]: { held: ['fluxweb_a2', 'fluxdb_ab'] } }); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); + }); + + describe('a peer that does not answer', () => { + // This node's syncthing saw the peer after it started, and holds no live + // connection to it for whichever folders the test names. + const evidence = (connectedFolders) => { + sinon.stub(axios, 'get').rejects(new Error('connect ECONNREFUSED')); + sinon.stub(syncthingService, 'getConfigDevices').resolves([{ name: PEER, deviceID: 'PEERDEVICE' }]); + sinon.stub(syncthingService, 'getDbCompletion').callsFake(async ({ folder }) => ({ + remoteState: connectedFolders.includes(folder) ? 'valid' : 'notSharing', + })); + sinon.stub(syncthingService, 'getDeviceStats').resolves({ PEERDEVICE: { lastSeen: '2026-10-16T10:00:00Z' } }); + sinon.stub(syncthingService, 'getSystemStatus').resolves({ startTime: '2026-10-16T09:00:00Z' }); + sinon.stub(registryManager, 'shuttingDownNodes').resolves([]); + }; + + beforeEach(() => { + globalState.syncthingDevicesIDCache.clear(); + }); + + it('frees the app when its connection is gone for every g: folder', async () => { + evidence([]); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.NOT_RUNNING); + }); + + // Canary for the test above: the same peer with one folder still connected + // is one that may still be writing it. + it('does not free the app while one g: folder is still connected to it', async () => { + evidence(['fluxweb_a']); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); + }); + + it('does not free the app while the anchor\'s folder is still connected to it', async () => { + evidence(['fluxdb_a']); + + expect(await peerComponentState(PEER, appCtx)).to.equal(PeerComponent.UNKNOWN); + }); + }); + + describe('peerUndecided', () => { + const answers = (decided) => { + peerIdentityService.askSigned.resolves({ + verdict: peerIdentityService.IdentityVerdict.VERIFIED, answer: { held: decided, decided }, + }); + }; + + it('reads a peer that has decided any of them as decided', async () => { + answers(['fluxweb_a']); + + expect(await peerUndecided(PEER, appCtx)).to.equal(false); + }); + + it('reads a peer that has decided none of them as undecided', async () => { + answers(['fluxweb_b']); + + expect(await peerUndecided(PEER, appCtx)).to.equal(true); + }); + }); + }); }); diff --git a/tests/unit/peerFolderLiveness.test.js b/tests/unit/peerFolderLiveness.test.js index 14a6fdbd81..1e05c5444e 100644 --- a/tests/unit/peerFolderLiveness.test.js +++ b/tests/unit/peerFolderLiveness.test.js @@ -50,7 +50,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, misrouted: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); expect(probes(), 'what another node holds is not this peer\'s answer').to.equal(0); }); @@ -94,7 +94,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, unproven: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); sinon.assert.notCalled(axiosMock.get); }); @@ -108,7 +108,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); @@ -188,7 +188,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); sinon.assert.notCalled(axiosMock.get); }); @@ -334,7 +334,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: false, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); @@ -350,7 +350,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: false, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); @@ -387,6 +387,16 @@ describe('peerFolderLiveness', () => { expect(answer.ready).to.be.false; }); + it('reads the folders a peer cannot seed because their volume will not mount, and none from a peer that does not say', async () => { + answers({ data: { data: { ready: true, folders: [], heldOut: ['fluxa_a'], volumeGone: ['fluxb_b'] } } }); + const saying = await createPeerFolderLiveness().read('10.0.0.2:16127'); + answers({ data: { data: { ready: true, folders: [] } } }); + const silent = await createPeerFolderLiveness().read('10.0.0.2:16127'); + + expect([saying.heldOut, saying.volumeGone]).to.deep.equal([['fluxa_a'], ['fluxb_b']]); + expect([silent.heldOut, silent.volumeGone]).to.deep.equal([[], []]); + }); + it('treats a malformed body as reachable but not ready', async () => { answers({ data: {} }); const liveness = createPeerFolderLiveness(); @@ -394,7 +404,7 @@ describe('peerFolderLiveness', () => { const answer = await liveness.read('10.0.0.2:16127'); expect(answer).to.deep.equal({ - reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, + reachable: true, answerable: true, ready: false, folders: [], holding: {}, seeding: {}, notLookedAt: [], heldOut: [], volumeGone: [], }); }); diff --git a/tests/unit/peerNotification.test.js b/tests/unit/peerNotification.test.js index 45bd5a965b..37e46bd94b 100644 --- a/tests/unit/peerNotification.test.js +++ b/tests/unit/peerNotification.test.js @@ -17,8 +17,12 @@ describe('peerNotification tests', () => { // same object. let departingApps; let testInstallingApps; + let unconfirmedApps; let announceCycle; let findOneInDatabaseStub; + let placementsOfStub; + let waitForBackfillStub; + let waitForLocalSocketAddressStub; // One stub map, so a test that needs a different interval, expiry or cycle // length states only that difference instead of restating sixty lines. @@ -55,7 +59,7 @@ describe('peerNotification tests', () => { nodeTier: sinon.stub().resolves('cumulus'), }, '../fluxNetworkHelper': { - getLocalSocketAddress: sinon.stub().resolves('192.168.1.1:16127'), + waitForLocalSocketAddress: waitForLocalSocketAddressStub, }, '../geolocationService': { isStaticIP: sinon.stub().returns(true), @@ -109,10 +113,12 @@ describe('peerNotification tests', () => { '../utils/globalState': { departingApps, testInstallingApps, + unconfirmedApps, announceCycle, runningAppsCache: new Set(), }, '../utils/nodeSigner': { nodeSigner: nodeSignerStub }, + '../appManagement/appPlacements': { placementsOf: placementsOfStub, waitForBackfill: waitForBackfillStub }, '../../lib/log': logStub, }); @@ -124,15 +130,19 @@ describe('peerNotification tests', () => { }; findOneInDatabaseStub = sinon.stub().resolves(null); + // every installed app has a record unless a test says otherwise + placementsOfStub = sinon.stub().callsFake(async (names) => new Map(names.map((name) => [name, new Date('2026-09-01T00:00:00.000Z')]))); enqueueAllStub = sinon.stub().resolves(); waitForBootDrainSettledStub = sinon.stub().resolves(); + waitForBackfillStub = sinon.stub().resolves(); + waitForLocalSocketAddressStub = sinon.stub().resolves('192.168.1.1:16127'); storeAppRunningMessageStub = sinon.stub().resolves(); storeAppStateEventStub = sinon.stub().resolves(); broadcastMessageToAllStub = sinon.stub().resolves('signed'); nodeSignerStub = sinon.stub().resolves({ pubKey: '04', sign: () => 'sig' }); installedAppsStub = sinon.stub().resolves({ status: 'success', - data: [{ name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], + data: [{ name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], }); // The REAL departing tracker, taken fresh per test rather than // reimplemented here: a fake that counts differently from the module would @@ -141,7 +151,9 @@ describe('peerNotification tests', () => { // this suite's alone - evicting the entry instead would hand another object // to every module loaded after it, and globalState is a singleton whose // flags decide whether an operation may start at all. - ({ departingApps, testInstallingApps, announceCycle } = proxyquire('../../ZelBack/src/services/utils/globalState', {})); + ({ + departingApps, testInstallingApps, unconfirmedApps, announceCycle, + } = proxyquire('../../ZelBack/src/services/utils/globalState', {})); peerNotification = loadPeerNotification(); }); @@ -193,8 +205,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, ], }); // only app1's container is running @@ -211,8 +223,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'app2', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, ], }); departingApps.enter('app2'); @@ -240,6 +252,40 @@ describe('peerNotification tests', () => { expect(message.apps.map((a) => a.name)).to.deep.equal(['app1']); }); + // Boot holds an app whose location record it could not read. Naming it would + // renew a record that may have expired, and the boot check would then read the + // renewed record as valid and start the app beside the one placed elsewhere. + it('does not announce an app boot has not yet confirmed it holds', async () => { + installedAppsStub.resolves({ + status: 'success', + data: [ + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + ], + }); + unconfirmedApps.set('app2', Symbol('app2')); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect( + message.apps.map((a) => a.name), + 'the unconfirmed app is excluded and the rest still announced', + ).to.deep.equal(['app1']); + }); + + it('announces the app again once boot has confirmed it', async () => { + unconfirmedApps.set('app1', Symbol('app1')); + await peerNotification.checkAndNotifyPeersOfRunningApps(); + expect(storeAppRunningMessageStub.called, 'announced while unconfirmed').to.be.false; + + unconfirmedApps.delete('app1'); + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps.map((a) => a.name)).to.deep.equal(['app1']); + }); + it('announces nothing when every installed app is departing', async () => { departingApps.enter('app1'); @@ -266,7 +312,7 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, { name: 'sealed', version: 8, enterprise: 'encrypted-blob', compose: [], hash: 'h2' }, ], }); @@ -306,8 +352,8 @@ describe('peerNotification tests', () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'trialapp', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'trialapp', hash: 'trialapphash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, ], }); testInstallingApps.add('trialapp'); @@ -321,6 +367,97 @@ describe('peerNotification tests', () => { ).to.deep.equal(['app1']); }); + describe('when this node took each app', () => { + it('announces the time from its placement record, not from its location row', async () => { + placementsOfStub.resolves(new Map([['app1', new Date('2026-06-01T12:00:00.000Z')]])); + findOneInDatabaseStub.resolves({ runningSince: new Date('2026-10-21T02:08:56.000Z') }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps).to.deep.equal([{ name: 'app1', hash: 'app1hash', runningSince: '2026-06-01T12:00:00.000Z' }]); + expect(findOneInDatabaseStub.called, 'the location row is not read for a recorded app').to.equal(false); + }); + + it('keeps announcing the recorded time when its own location row has lapsed', async () => { + placementsOfStub.resolves(new Map([['app1', new Date('2026-06-01T12:00:00.000Z')]])); + findOneInDatabaseStub.resolves(null); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps[0].runningSince).to.equal('2026-06-01T12:00:00.000Z'); + }); + + it('announces an app with no record from its location row, and says so', async () => { + placementsOfStub.resolves(new Map()); + findOneInDatabaseStub.resolves({ runningSince: new Date('2026-07-01T00:00:00.000Z') }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps[0].runningSince).to.equal('2026-07-01T00:00:00.000Z'); + expect(logStub.warn.calledWithMatch(/no placement record for app1/)).to.equal(true); + }); + + it('announces an app with neither a record nor a row as placed now', async () => { + placementsOfStub.resolves(new Map()); + findOneInDatabaseStub.resolves(null); + const before = Date.now(); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(new Date(message.apps[0].runningSince).getTime()).to.be.within(before, Date.now()); + }); + }); + + // No node stores an entry without a name and hash, and a node that has not + // upgraded refuses the whole message for one - which would cost this node + // every other app it holds. + it('leaves out an installed app without a hash and announces the rest', async () => { + installedAppsStub.resolves({ + status: 'success', + data: [ + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'probe', version: 8, compose: [{ name: 'web', containerData: '/data' }] }, + { name: 'app2', hash: 'app2hash', version: 4, compose: [{ name: 'c2', containerData: '/data' }] }, + ], + }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.apps.map((a) => a.name)).to.deep.equal(['app1', 'app2']); + expect(broadcastMessageToAllStub.firstCall.args[0]).to.equal(message); + expect(logStub.warn.calledWithMatch(/not announced: probe/)).to.equal(true); + }); + + it('announces nothing when no installed app has a hash', async () => { + installedAppsStub.resolves({ + status: 'success', + data: [{ name: 'probe', version: 8, compose: [{ name: 'web', containerData: '/data' }] }], + }); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + expect(storeAppRunningMessageStub.called).to.equal(false); + expect(broadcastMessageToAllStub.called).to.equal(false); + }); + + // Every peer applies the store's rule, so a message this node refuses is one + // no peer keeps either. + it('broadcasts nothing its own store refuses', async () => { + storeAppRunningMessageStub.resolves(new Error('Invalid Flux App Running message: apps array exceeds maxAppsPerNode')); + + await peerNotification.checkAndNotifyPeersOfRunningApps(); + + expect(storeAppRunningMessageStub.calledOnce).to.equal(true); + expect(broadcastMessageToAllStub.called, 'a refused announcement is not sent').to.equal(false); + expect(storeAppStateEventStub.called).to.equal(false); + expect(logStub.error.calledWithMatch(/refuses its own announcement/)).to.equal(true); + }); + // An empty snapshot must NEVER be broadcast: on the receive side an empty v2 // message deletes every appsLocations row for the sender's IP - and the sender // stores its own message first, so it erases its own network presence. The @@ -365,15 +502,64 @@ describe('peerNotification tests', () => { expect(message.apps.map((a) => a.name)).to.deep.equal(['app1']); }); + it('reads no placement record before every app installed at start has one', async () => { + let openBackfillGate; + waitForBackfillStub.callsFake(() => new Promise((resolve) => { openBackfillGate = resolve; })); + + const callPromise = peerNotification.checkAndNotifyPeersOfRunningApps(); + await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(placementsOfStub.called, 'read the records before the backfill finished').to.be.false; + expect(broadcastMessageToAllStub.called).to.be.false; + + openBackfillGate(); + await callPromise; + expect(placementsOfStub.calledOnce).to.be.true; + expect(broadcastMessageToAllStub.calledOnce).to.be.true; + }); + + it('announces as soon as this node learns its address', async () => { + let learnAddress; + waitForLocalSocketAddressStub.callsFake(() => new Promise((resolve) => { learnAddress = resolve; })); + + const callPromise = peerNotification.checkAndNotifyPeersOfRunningApps(); + await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(broadcastMessageToAllStub.called, 'broadcast before the node knew its address').to.be.false; + + learnAddress('192.168.1.1:16127'); + await callPromise; + expect(broadcastMessageToAllStub.calledOnce).to.be.true; + const [message] = storeAppRunningMessageStub.firstCall.args; + expect(message.ip).to.equal('192.168.1.1:16127'); + }); + + // A removal waits for a cycle in flight; a cycle still waiting for the node's + // address has taken no snapshot and holds up no removal. + it('is not in flight while it waits for the node\'s address', async () => { + let learnAddress; + waitForLocalSocketAddressStub.callsFake(() => new Promise((resolve) => { learnAddress = resolve; })); + + const callPromise = peerNotification.checkAndNotifyPeersOfRunningApps(); + await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(announceCycle.locked, 'in flight while waiting for the address').to.equal(false); + + learnAddress('192.168.1.1:16127'); + await callPromise; + expect(announceCycle.locked).to.equal(false); + expect(broadcastMessageToAllStub.calledOnce).to.be.true; + }); + // g:/r: apps needed a carve-out while the set came from run-state, because a // masterSlave app stops its slave components deliberately. They need none now. it('announces g:/r: apps with stopped components, without a carve-out', async () => { installedAppsStub.resolves({ status: 'success', data: [ - { name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, - { name: 'gapp', version: 4, compose: [{ name: 'gc', containerData: 'g:/data' }] }, - { name: 'rapp', version: 4, compose: [{ name: 'rc', containerData: 'r:/data' }] }, + { name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }, + { name: 'gapp', hash: 'gapphash', version: 4, compose: [{ name: 'gc', containerData: 'g:/data' }] }, + { name: 'rapp', hash: 'rapphash', version: 4, compose: [{ name: 'rc', containerData: 'r:/data' }] }, ], }); // neither gapp's nor rapp's containers are running @@ -409,7 +595,7 @@ describe('peerNotification tests', () => { if (opts.cycleMs && cycles <= (opts.slowCycles ?? 1)) clock.tick(opts.cycleMs); return { status: 'success', - data: [{ name: 'app1', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], + data: [{ name: 'app1', hash: 'app1hash', version: 4, compose: [{ name: 'c1', containerData: '/data' }] }], }; }); const mod = loadPeerNotification({ ...opts, installedApps }); diff --git a/tests/unit/placementsMigration.test.js b/tests/unit/placementsMigration.test.js new file mode 100644 index 0000000000..e75f504b45 --- /dev/null +++ b/tests/unit/placementsMigration.test.js @@ -0,0 +1,41 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('placements migration', () => { + let migration; + let learnAddress; + let findInDatabaseStub; + let backfillPlacementsStub; + + beforeEach(() => { + findInDatabaseStub = sinon.stub().resolves([{ name: 'web' }, { name: 'db' }]); + backfillPlacementsStub = sinon.stub().resolves({ created: 2, fromLocationRows: 2 }); + migration = proxyquire('../../ZelBack/src/services/migrations/placements', { + config: { database: { appslocal: { database: 'localzelapps' } } }, + '../dbHelper': { + databaseConnection: () => ({ db: () => ({}) }), + findInDatabase: findInDatabaseStub, + }, + '../fluxNetworkHelper': { + waitForLocalSocketAddress: () => new Promise((resolve) => { learnAddress = resolve; }), + }, + '../appManagement/appPlacements': { backfillPlacements: backfillPlacementsStub }, + '../utils/appConstants': { localAppsInformation: 'zelappsinformation' }, + }); + }); + + // The rows a record is dated from are filed under this node's address, so no + // record is written before the node knows it. + it('writes nothing until this node knows its address, then backfills the installed apps from it', async () => { + const running = migration.backfillPlacements(); + await new Promise((resolve) => { setImmediate(resolve); }); + sinon.assert.notCalled(backfillPlacementsStub); + + learnAddress('203.0.113.40:16127'); + await running; + + sinon.assert.calledOnceWithExactly(backfillPlacementsStub, ['web', 'db'], '203.0.113.40:16127'); + expect(findInDatabaseStub.firstCall.args[1]).to.equal('zelappsinformation'); + }); +}); diff --git a/tests/unit/portManager.test.js b/tests/unit/portManager.test.js index 08078917e1..daf6d4559e 100644 --- a/tests/unit/portManager.test.js +++ b/tests/unit/portManager.test.js @@ -12,11 +12,21 @@ const appUninstaller = require('../../ZelBack/src/services/appLifecycle/appUnins const networkStateService = require('../../ZelBack/src/services/networkStateService'); const { EventEmitter } = require('node:events'); const fluxHttpTestServer = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); +const fluxUdpTestServer = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); +const policyStore = require('../../ZelBack/src/services/policyStore'); const { requireMongo } = require('./dbTestHelper'); const appQueryService = require('../../ZelBack/src/services/appQuery/appQueryService'); const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); const globalState = require('../../ZelBack/src/services/utils/globalState'); +// what upnpService.mapUpnpPort answers for a port mapped, and for one the router refused +const MAPPED = { + mapped: true, code: null, reason: null, routerWide: false, +}; +const REFUSED = { + mapped: false, code: 728, reason: 'TCP 31000 not mapped: UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)', routerWide: true, +}; + describe('portManager tests', () => { before(requireMongo); @@ -606,7 +616,7 @@ describe('portManager tests', () => { sinon.stub(upnpService, 'isUPNP').returns(false); sinon.stub(fluxNetworkHelper, 'isFirewallActive').resolves(false); sinon.stub(fluxNetworkHelper, 'allowPort').resolves(true); - sinon.stub(upnpService, 'mapUpnpPort').resolves(true); + sinon.stub(upnpService, 'mapUpnpPort').resolves(MAPPED); sinon.stub(serviceHelper, 'delay').resolves(); sinon.stub(appUninstaller, 'removeAppLocally').resolves(); portManager.upnpMapFailures.clear(); @@ -643,7 +653,7 @@ describe('portManager tests', () => { // removeAppLocally(force, sendMessage) - a transient router blip nuked // a running app and broadcast its removal to the network upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); @@ -653,8 +663,8 @@ describe('portManager tests', () => { it('should retry a failed port within the cycle and record no failure on recovery', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.onFirstCall().resolves(false); - upnpService.mapUpnpPort.resolves(true); + upnpService.mapUpnpPort.onFirstCall().resolves(REFUSED); + upnpService.mapUpnpPort.resolves(MAPPED); await portManager.restoreAppsPortsSupport(); @@ -664,7 +674,7 @@ describe('portManager tests', () => { it('should not remove before the sustained window even after enough failing cycles', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); await portManager.restoreAppsPortsSupport(); @@ -677,7 +687,7 @@ describe('portManager tests', () => { it('should remove and broadcast only after sustained failure (cycles AND window)', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); const nowMonotonicMs = Number(process.hrtime.bigint() / 1000000n); portManager.upnpMapFailures.set('App1', { cycles: 2, @@ -696,7 +706,7 @@ describe('portManager tests', () => { { name: 'App2', version: 3, ports: [30002] }, ]); upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); @@ -710,11 +720,11 @@ describe('portManager tests', () => { it('should clear the failure tracker once mapping succeeds again', async () => { upnpService.isUPNP.returns(true); - upnpService.mapUpnpPort.resolves(false); + upnpService.mapUpnpPort.resolves(REFUSED); await portManager.restoreAppsPortsSupport(); expect(portManager.upnpMapFailures.get('App1').cycles).to.equal(1); - upnpService.mapUpnpPort.resolves(true); + upnpService.mapUpnpPort.resolves(MAPPED); await portManager.restoreAppsPortsSupport(); expect(portManager.upnpMapFailures.has('App1')).to.be.false; @@ -1190,6 +1200,8 @@ describe('portManager tests', () => { // one witness to accept, two to refuse - and where the exits that skip them are. describe('checkInstallingAppPortAvailable decides on every way of running out', () => { let port; + let token; + let udpServers; const PEERS = ['10.0.0.1:16127', '10.0.0.2:16127', '10.0.0.3:16127', '10.0.0.4:16127', '10.0.0.5:16127']; @@ -1217,12 +1229,20 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', }); beforeEach(() => { - sinon.stub(fluxHttpTestServer, 'FluxHttpTestServer').callsFake(() => { + token = null; + udpServers = []; + sinon.stub(fluxHttpTestServer, 'FluxHttpTestServer').callsFake((runToken) => { + token = runToken; const server = new EventEmitter(); server.listen = () => { setImmediate(() => server.emit('listening')); return server; }; server.close = (callback) => { if (callback) setImmediate(callback); return server; }; return server; }); + sinon.stub(fluxUdpTestServer, 'FluxUdpTestServer').callsFake(() => { + const server = { listen: sinon.stub().resolves(), allow: sinon.stub(), close: sinon.stub() }; + udpServers.push(server); + return server; + }); sinon.stub(serviceHelper, 'delay').resolves(); sinon.stub(fluxNetworkHelper, 'getLocalSocketAddress').resolves('1.2.3.4:16127'); sinon.stub(fluxNetworkHelper, 'getFluxNodePublicKey').resolves('04pubkey'); @@ -1235,6 +1255,7 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', afterEach(() => { sinon.restore(); + fluxNetworkHelper.setUdpReachability(null); }); // One attempt per drawn peer: the observer is drawn first, so the attempt @@ -1248,12 +1269,13 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', }); sinon.stub(axios, 'post').callsFake(async () => { - const answer = answers[attempt]; + // An answer that carries our token is written against the run's token. + const answer = typeof answers[attempt] === 'function' ? answers[attempt](token) : answers[attempt]; if (!answer || answer === UNREACHABLE) throw new Error('connect ECONNREFUSED'); return answer; }); - return portManager.checkInstallingAppPortAvailable([port]); + return portManager.checkInstallingAppPortAvailable([port], 'web'); }; // The bug: a verdict and "nobody has decided" were the same value, so a last @@ -1281,9 +1303,9 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', await withPeers([UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE]); - sinon.assert.calledOnceWithExactly(allowTest, port); - sinon.assert.calledWith(deleteTest, port); - expect(deleteTest.firstCall.calledAfter(allowTest.firstCall)).to.equal(true); + expect(allowTest.args).to.deep.equal([[port, 'tcp'], [port, 'udp']]); + expect(deleteTest.args).to.deep.equal([[port, 'tcp'], [port, 'udp']]); + expect(deleteTest.firstCall.calledAfter(allowTest.lastCall)).to.equal(true); sinon.assert.notCalled(allowBoth); sinon.assert.notCalled(deleteBoth); }); @@ -1358,4 +1380,273 @@ describe('checkInstallingAppPortAvailable decides on every way of running out', expect(result.reason).to.equal('noOtherObserver'); expect(result.peers).to.have.length(1); }); + + it('asks no peer, and removes what the router kept, when the router refuses the port under test', async () => { + upnpService.isUPNP.returns(true); + sinon.stub(upnpService, 'mapUpnpPort').resolves(REFUSED); + const removeMapping = sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); + + const result = await withPeers([UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE]); + + expect(result).to.deep.equal({ ok: false, reason: 'error' }); + sinon.assert.notCalled(axios.post); + sinon.assert.calledWith(removeMapping, port, 'Flux_App_web'); + }); + + it('maps the port under test under the application\'s own description, the one it installs under', async () => { + upnpService.isUPNP.returns(true); + const mapUpnpPort = sinon.stub(upnpService, 'mapUpnpPort').resolves(REFUSED); + sinon.stub(upnpService, 'removeMapUpnpPort').resolves(true); + + await withPeers([UNREACHABLE]); + + sinon.assert.calledOnceWithExactly(mapUpnpPort, port, 'Flux_App_web'); + }); + + // What a peer that probes UDP answers. `udp` is the port's UDP reply: our + // token's reply, null for silence, or absent for a port it could not send to. + const answering = (udp) => (runToken) => ({ + data: { + status: 'success', + data: { + answered: { [port]: `HTTP/1.1 200 OK\r\nX-Flux-Port-Test: ${runToken}\r\n\r\n` }, + udpAnswered: udp === undefined ? {} : { [port]: udp === 'ours' ? fluxUdpTestServer.udpReply(runToken).toString() : udp }, + }, + }, + }); + // A peer on code that reads TCP and does not probe UDP. + const tcpOnly = (runToken) => ({ + data: { status: 'success', data: { answered: { [port]: `X-Flux-Port-Test: ${runToken}` } } }, + }); + + it('proves the ports when one peer reads our token over TCP and hears it over UDP', async () => { + const result = await withPeers([answering('ours')]); + + expect(result).to.deep.equal({ ok: true, reason: 'proven', port: null }); + sinon.assert.calledOnce(axios.post); + }); + + it('asks the peer to probe UDP, and opens a UDP test server on every port', async () => { + await withPeers([answering('ours')]); + + expect(JSON.parse(axios.post.firstCall.args[1]).protocols).to.deep.equal(['tcp', 'udp']); + expect(udpServers).to.have.length(1); + sinon.assert.calledWith(udpServers[0].listen, port); + sinon.assert.calledOnce(udpServers[0].close); + }); + + // A UDP reply goes wherever a probe claims to come from; answering anyone + // but the peer being asked would make the port under test a reflector. + it('answers only the peer it is asking, from before it asks', async () => { + await withPeers([answering(null), answering('ours')]); + + expect(udpServers[0].allow.args).to.deep.equal([['10.0.0.1'], ['10.0.0.2']]); + expect(udpServers[0].allow.secondCall.calledBefore(axios.post.secondCall)).to.equal(true); + }); + + it('refuses once two distinct peers hear nothing over UDP and no peer proves it', async () => { + const result = await withPeers([answering(null), answering(null), UNREACHABLE, UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'two peers hearing nothing did not refuse').to.equal(false); + expect(result.reason).to.equal('udpUnreachable'); + expect(result.port).to.equal(port); + expect(result.peers).to.deep.equal(['10.0.0.1', '10.0.0.2']); + }); + + it('refuses on a UDP answer that is not our token, as on silence', async () => { + const result = await withPeers([answering('somebody else'), answering(null)]); + + expect(result.ok).to.equal(false); + expect(result.reason).to.equal('udpUnreachable'); + }); + + // A peer that cannot hear UDP replies is silent for every node. Two of them + // first do not refuse while a peer that can prove the port is still to ask. + it('asks every attempt left for proof after two peers hear nothing', async () => { + const result = await withPeers([answering(null), answering(null), answering('ours')]); + + expect(result).to.deep.equal({ ok: true, reason: 'proven', port: null }); + sinon.assert.callCount(axios.post, 3); + }); + + // UDP loses datagrams. Our token heard by one peer is proof, and another + // peer's silence on the same port is loss. + it('takes our token heard by one peer over silence heard by another', async () => { + const result = await withPeers([answering(null), answering('ours'), answering(null)]); + + expect(result).to.deep.equal({ ok: true, reason: 'proven', port: null }); + }); + + it('proceeds without proof over UDP when one peer alone heard nothing', async () => { + const result = await withPeers([answering(null), UNREACHABLE, UNREACHABLE, UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'refused on a single UDP witness').to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + expect(result.port).to.equal(port); + expect(result.peers).to.deep.equal(['10.0.0.1']); + }); + + // A peer that could not send says nothing about the port. + it('counts a port the peer could not send to as no evidence', async () => { + const result = await withPeers([answering(undefined), answering(undefined), answering(undefined), answering(undefined), answering(undefined)]); + + expect(result.ok).to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + expect(result.peers).to.deep.equal([]); + }); + + // While the fleet carries peers that do not probe UDP, an install goes ahead + // on TCP proof, having asked every peer it may for UDP. + it('proceeds on TCP proof when no peer asked probes UDP', async () => { + const result = await withPeers([tcpOnly, tcpOnly, tcpOnly, tcpOnly, tcpOnly]); + + expect(result.ok).to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + sinon.assert.callCount(axios.post, 5); + }); + + // Our token over TCP is proof; peers that later fail to reach the port over + // TCP cannot outweigh it. + it('lets no later peers that could not reach the port outweigh TCP proof', async () => { + const result = await withPeers([tcpOnly, couldNotReach(port), couldNotReach(port), UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'TCP proof was outweighed').to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + }); + + it('lets no later peers reading something else outweigh TCP proof', async () => { + const result = await withPeers([tcpOnly, answeringSomethingElse(port), answeringSomethingElse(port), UNREACHABLE, UNREACHABLE]); + + expect(result.ok, 'TCP proof was outweighed').to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + }); + + // Two ports. The first peer hears nothing on either; the second proves the + // first port and hears nothing on the second. Both peers are silent on the + // second port, and the first peer's silence on the first is withdrawn. + it('corroborates on the port two peers heard nothing on, withdrawing what was proven', async () => { + const other = port + 1; + const two = (udp) => (runToken) => ({ + data: { + status: 'success', + data: { + answered: { [port]: `X-Flux-Port-Test: ${runToken}`, [other]: `X-Flux-Port-Test: ${runToken}` }, + udpAnswered: Object.fromEntries(Object.entries(udp).map(([p, v]) => [p, v === 'ours' ? fluxUdpTestServer.udpReply(runToken).toString() : v])), + }, + }, + }); + let attempt = -1; + sinon.stub(networkStateService, 'getRandomExternalObserver').callsFake(async () => { attempt += 1; return PEERS[attempt] || null; }); + const answers = [two({ [port]: null, [other]: null }), two({ [port]: 'ours', [other]: null })]; + sinon.stub(axios, 'post').callsFake(async () => { + if (!answers[attempt]) throw new Error('connect ECONNREFUSED'); + return answers[attempt](token); + }); + + const result = await portManager.checkInstallingAppPortAvailable([port, other], 'web'); + + expect(result.ok).to.equal(false); + expect(result.reason).to.equal('udpUnreachable'); + expect(result.port).to.equal(other); + expect(result.readings).to.deep.equal({ '10.0.0.1': other, '10.0.0.2': other }); + }); + + // A port answered over TCP by something else is forwarded elsewhere; what the + // same peer heard over UDP describes that forward, not this node's UDP. + it('refuses as notOurs, not udpUnreachable, when the peers that hear nothing read TCP as someone else\'s', async () => { + const elsewhere = () => ({ + data: { + status: 'success', + data: { answered: { [port]: 'HTTP/1.1 200 OK\r\n\r\nsomebody else' }, udpAnswered: { [port]: null } }, + }, + }); + + const result = await withPeers([elsewhere, elsewhere, UNREACHABLE]); + + expect(result.reason).to.equal('notOurs'); + expect(result.udpReadings).to.deep.equal({ '10.0.0.1': 'silent', '10.0.0.2': 'silent' }); + expect(fluxNetworkHelper.getUdpReachability(), 'reported as this node\'s UDP').to.equal(null); + }); + + it('reports UDP reachable in health on a proof', async () => { + await withPeers([answering('ours')]); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: true }); + }); + + // A kill switch, not an opt-in: absent, the refusal stands. + it('installs as udpUnproven, still reporting UDP in health, when the policy turns the refusal over UDP off', async () => { + sinon.stub(policyStore, 'getDocument').withArgs('udpinstallrefusaloff').returns(true); + + const result = await withPeers([answering(null), answering(null), UNREACHABLE]); + + expect(result.ok).to.equal(true); + expect(result.reason).to.equal('udpUnproven'); + expect(result.peers).to.deep.equal(['10.0.0.1', '10.0.0.2']); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: false, port, peers: ['10.0.0.1', '10.0.0.2'] }); + }); + + it('refuses when the policy carries the switch as anything but true', async () => { + sinon.stub(policyStore, 'getDocument').withArgs('udpinstallrefusaloff').returns(false); + + const result = await withPeers([answering(null), answering(null), UNREACHABLE]); + + expect(result.reason).to.equal('udpUnreachable'); + }); + + it('reports the refused port and its peers in health when two peers hear nothing over UDP', async () => { + await withPeers([answering(null), answering(null), UNREACHABLE]); + expect(fluxNetworkHelper.getUdpReachability()).to.deep.equal({ reachable: false, port, peers: ['10.0.0.1', '10.0.0.2'] }); + }); + + it('closes its UDP test servers when the test fails', async () => { + fluxNetworkHelper.getLocalSocketAddress.resolves('1.2.3.4:16127'); + fluxUdpTestServer.FluxUdpTestServer.callsFake(() => { + const server = { listen: sinon.stub().rejects(new Error('EADDRINUSE')), allow: sinon.stub(), close: sinon.stub() }; + udpServers.push(server); + return server; + }); + + const result = await withPeers([answering('ours')]); + + expect(result).to.deep.equal({ ok: false, reason: 'error' }); + sinon.assert.calledOnce(udpServers[0].close); + }); +}); + +describe('udpPortsNotOurs and udpDisputes', () => { + const TOKEN = 'a'.repeat(32); + const ours = fluxUdpTestServer.udpReply(TOKEN).toString(); + + it('adds every port our token came back from to the proven set', () => { + const proven = new Set(); + expect(portManager.udpPortsNotOurs([31000, 31001], { 31000: ours, 31001: ours }, TOKEN, proven)).to.deep.equal([]); + expect([...proven]).to.deep.equal([31000, 31001]); + }); + + it('names every port this peer heard nothing of ours on', () => { + expect(portManager.udpPortsNotOurs([31000, 31001, 31002], { 31000: ours, 31001: null, 31002: 'other' }, TOKEN, new Set())) + .to.deep.equal([31001, 31002]); + }); + + it('reads a port missing from the answers as no evidence', () => { + expect(portManager.udpPortsNotOurs([31000], {}, TOKEN, new Set())).to.deep.equal([]); + }); + + it('skips ports outside the app port range, which the peer does not probe', () => { + const proven = new Set(); + expect(portManager.udpPortsNotOurs([0], { 0: null }, TOKEN, proven)).to.deep.equal([]); + expect(portManager.everyUdpPortProven([0], proven)).to.equal(true); + }); + + it('reads one port\'s UDP answer as ours, silent, another\'s answer, or not probed', () => { + expect(portManager.udpReadingOf({ 31000: ours }, 31000, TOKEN)).to.equal('ours'); + expect(portManager.udpReadingOf({ 31000: null }, 31000, TOKEN)).to.equal('silent'); + expect(portManager.udpReadingOf({ 31000: 'hello' }, 31000, TOKEN)).to.equal('otherAnswer'); + expect(portManager.udpReadingOf({}, 31000, TOKEN)).to.equal('notProbed'); + }); + + it('keeps a peer as a witness only while a port it heard nothing on is unproven', () => { + const disagreements = new Map([['a', [31000, 31001]], ['b', [31000]]]); + expect(portManager.udpDisputes(disagreements, new Set([31000]))).to.deep.equal(new Map([['a', 31001]])); + }); }); diff --git a/tests/unit/portTestEcho.test.js b/tests/unit/portTestEcho.test.js index 9cc1042200..124e25b3e5 100644 --- a/tests/unit/portTestEcho.test.js +++ b/tests/unit/portTestEcho.test.js @@ -1,10 +1,12 @@ const http = require('node:http'); const { EventEmitter } = require('node:events'); const { expect } = require('chai'); +const config = require('config'); const sinon = require('sinon'); const verificationHelper = require('../../ZelBack/src/services/verificationHelper'); const fluxCommunicationUtils = require('../../ZelBack/src/services/fluxCommunicationUtils'); const { FluxHttpTestServer } = require('../../ZelBack/src/services/utils/fluxHttpTestServer'); +const { FluxUdpTestServer } = require('../../ZelBack/src/services/utils/fluxUdpTestServer'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); const portManager = require('../../ZelBack/src/services/appNetwork/portManager'); const { Privilege } = require('../../ZelBack/src/services/utils/privileges'); @@ -96,6 +98,92 @@ describe('the pre-install port test, end to end', () => { }); }); +// The UDP half, end to end: this node's UDP test server answers a probe with +// the run's secret, the peer probes and hands back what replied, and this node +// compares. +describe('the pre-install port test over UDP, end to end', () => { + const TOKEN = 'f00dcafe'.repeat(4); + // Short enough that silence costs the suite little; the schedule's shape is + // the default's. + const QUICK = { schedule: [0, 50, 150], deadline: 400 }; + let server; + let port; + + beforeEach(async () => { + server = new FluxUdpTestServer(TOKEN); + await server.listen(0, '127.0.0.1'); + ({ port } = server.address()); + }); + + afterEach(() => server.close()); + + it('proves the port when our own test server is what answers', async () => { + server.allow('127.0.0.1'); + + const answer = await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', port, QUICK); + + expect(answer, 'the peer heard nothing back').to.be.a('string'); + const proven = new Set(); + expect(portManager.udpPortsNotOurs([port], { [port]: answer }, TOKEN, proven)).to.deep.equal([]); + expect(proven.has(port)).to.equal(true); + }); + + it('hears nothing from a test server answering another peer', async () => { + server.allow('198.51.100.7'); + + expect(await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', port, QUICK)).to.equal(null); + }); + + it('hears nothing from a port with nothing on it', async () => { + server.close(); + + expect(await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', port, QUICK)).to.equal(null); + }); + + // A datagram lost on the way is the normal case for UDP, not a closed port. + it('is answered by a later datagram when the first is lost', async () => { + const lossy = dgram.createSocket('udp4'); + await new Promise((resolve) => { lossy.bind(0, '127.0.0.1', resolve); }); + const lossyPort = lossy.address().port; + let seen = 0; + lossy.on('message', (message, rinfo) => { + seen += 1; + if (seen > 1) lossy.send(Buffer.from(`FLUXPT ${TOKEN}`), rinfo.port, rinfo.address); + }); + + const answer = await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', lossyPort, QUICK); + lossy.close(); + + expect(answer).to.include(TOKEN); + expect(seen).to.be.at.least(2); + }); + + // A NAT in front of the requester can rewrite the reply's source port, so + // the reply is taken from whatever port it comes from. + it('takes a reply that comes from another source port', async () => { + const probed = dgram.createSocket('udp4'); + const rewritten = dgram.createSocket('udp4'); + await new Promise((resolve) => { probed.bind(0, '127.0.0.1', resolve); }); + await new Promise((resolve) => { rewritten.bind(0, '127.0.0.1', resolve); }); + probed.on('message', (message, rinfo) => { + rewritten.send(Buffer.from(`FLUXPT ${TOKEN}`), rinfo.port, rinfo.address); + }); + + const probedPort = probed.address().port; + const replyPort = rewritten.address().port; + const answer = await fluxNetworkHelper.portAnsweredUdp('127.0.0.1', probedPort, QUICK); + probed.close(); + rewritten.close(); + + expect(replyPort, 'the reply came from the port probed').to.not.equal(probedPort); + expect(answer).to.include(TOKEN); + }); + + it('says nothing about a port it could not send to', async () => { + expect(await fluxNetworkHelper.portAnsweredUdp('::1', port, QUICK)).to.equal(undefined); + }); +}); + // The address this node connects to when a peer asks it to test some ports. // // It is NOT taken from the request body. A caller that names the address picks @@ -226,6 +314,153 @@ describe('checkAppAvailability probes the address that asked', () => { ); }); + it('probes each port over UDP when asked, handing back what replied', async () => { + const udpServer = new FluxUdpTestServer(TOKEN); + await udpServer.listen(port, HERE); + udpServer.allow(HERE); + + const answer = await ask( + { + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, protocols: ['tcp', 'udp'], + }, + HERE, + ); + udpServer.close(); + + expect(answer.status).to.equal('success'); + expect(answer.data.answered[port]).to.include(TOKEN); + expect(answer.data.udpAnswered[port]).to.include(TOKEN); + }); + + // The test server answers over IPv6 too, so TCP passes; a UDP probe is IPv4 + // and cannot be sent there. That says nothing about the port. + it('leaves out a port it could not send a UDP probe to', async () => { + const dualStack = new FluxHttpTestServer(TOKEN); + await new Promise((resolve) => dualStack.listen(0, '::1', resolve)); + const v6Port = dualStack.address().port; + + const answer = await ask( + { + ip: '::1', port: 16127, ports: [v6Port], pubKey: 'k', signature: 's', echo: true, protocols: ['tcp', 'udp'], + }, + '::1', + ); + await new Promise((resolve) => dualStack.close(resolve)); + + expect(answer.status).to.equal('success'); + expect(answer.data.answered[v6Port]).to.include(TOKEN); + expect(answer.data.udpAnswered).to.deep.equal({}); + }); + + it('probes UDP alone, reading nothing over TCP, for a caller that asks for UDP alone', async () => { + // Nothing listens on this port over TCP, so a TCP read would fail the test. + const udpServer = new FluxUdpTestServer(TOKEN); + await udpServer.listen(0, HERE); + udpServer.allow(HERE); + const udpOnly = udpServer.address().port; + const answer = await ask( + { + ip: HERE, port: 16127, ports: [udpOnly], pubKey: 'k', signature: 's', echo: true, protocols: ['udp'], + }, + HERE, + ); + udpServer.close(); + + expect(answer.status, 'the port was read over TCP').to.equal('success'); + expect(answer.data.udpAnswered[udpOnly]).to.include(TOKEN); + expect(answer.data).to.not.have.property('answered'); + }); + + it('refuses protocols other than tcp and udp', async () => { + const answer = await ask( + { + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, protocols: ['sctp'], + }, + HERE, + ); + + expect(answer.status).to.equal('error'); + expect(answer.data.message).to.include('tcp, udp or both'); + }); + + it('probes no UDP for a caller that does not ask', async () => { + const answer = await ask( + { + ip: HERE, port: 16127, ports: [port], pubKey: 'k', signature: 's', echo: true, + }, + HERE, + ); + + expect(answer.status).to.equal('success'); + expect(answer.data).to.not.have.property('udpAnswered'); + }); + + it('answers for every port when asked, a port that did not answer reading null', async () => { + const gone = http.createServer(); + await new Promise((resolve) => gone.listen(0, HERE, resolve)); + const closedPort = gone.address().port; + await new Promise((resolve) => gone.close(resolve)); + + const answer = await ask( + { + ip: HERE, port: 16127, ports: [closedPort, port], pubKey: 'k', signature: 's', echo: true, everyPort: true, + }, + HERE, + ); + + expect(answer.status, 'the test ended at the first port that failed').to.equal('success'); + expect(answer.data.answered[closedPort]).to.equal(null); + expect(answer.data.answered[port]).to.include(TOKEN); + }); + + it('ends at the first port that failed for a caller that does not ask for every port', async () => { + const gone = http.createServer(); + await new Promise((resolve) => gone.listen(0, HERE, resolve)); + const closedPort = gone.address().port; + await new Promise((resolve) => gone.close(resolve)); + + const answer = await ask( + { + ip: HERE, port: 16127, ports: [closedPort, port], pubKey: 'k', signature: 's', echo: true, + }, + HERE, + ); + + expect(answer.status).to.equal('error'); + expect(answer.data.message).to.include(`Failed port: ${closedPort}`); + }); + + // The largest app's ports, every one of them silent over UDP: each costs a + // probe's full deadline, and all of them have to be answered for inside the + // time the requester waits for this peer. + it('answers for 50 ports silent over UDP inside the requester\'s wait', async function () { + this.timeout(60000); + const servers = await Promise.all(Array.from({ length: 50 }, async () => { + const s = new FluxHttpTestServer(TOKEN); + await new Promise((resolve) => s.listen(0, HERE, resolve)); + return s; + })); + const ports = servers.map((s) => s.address().port); + + const started = Date.now(); + const answer = await ask( + { + ip: HERE, port: 16127, ports, pubKey: 'k', signature: 's', echo: true, protocols: ['tcp', 'udp'], + }, + HERE, + ); + const elapsed = Date.now() - started; + await Promise.all(servers.map((s) => new Promise((resolve) => s.close(resolve)))); + + // Nothing proves a port. A probe can land on another probe's socket here - + // requester and peer share this host - so an answer can be a probe rather + // than silence; it is never our token. + expect(answer.status).to.equal('success'); + expect(Object.values(answer.data.udpAnswered)).to.have.length(50); + expect(Object.values(answer.data.udpAnswered).every((reply) => reply === null || !reply.includes(TOKEN))).to.equal(true); + expect(elapsed, 'outlasted the requester\'s wait for a peer').to.be.below(config.fluxapps.portTestPeerTimeoutMs); + }); + // maxComponents (10) x ports per component (5) in appValidator. Each port // costs a connect timeout and they are tested in sequence. it('refuses more ports than an application could hold', async () => { diff --git a/tests/unit/registryManager.test.js b/tests/unit/registryManager.test.js index 1b9773af46..650b9fa136 100644 --- a/tests/unit/registryManager.test.js +++ b/tests/unit/registryManager.test.js @@ -1,7 +1,8 @@ +const { EventEmitter } = require('events'); const { expect } = require('chai'); const sinon = require('sinon'); const config = require('config'); -const { ObjectId } = require('mongodb'); +const { ObjectId, Collection } = require('mongodb'); const proxyquire = require('proxyquire'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); const { Privilege, authOf } = require('../../ZelBack/src/services/utils/privileges'); @@ -15,6 +16,7 @@ const verificationHelper = require('../../ZelBack/src/services/verificationHelpe const globalState = require('../../ZelBack/src/services/utils/globalState'); const localRemovalQueue = require('../../ZelBack/src/services/appLifecycle/localRemovalQueue'); const { appSyncEvents, EVENTS: SYNC_EVENTS } = require('../../ZelBack/src/services/utils/appSyncEvents'); +const { peerManager } = require('../../ZelBack/src/services/utils/peerState'); const { requireMongo } = require('./dbTestHelper'); describe('registryManager tests', () => { @@ -1092,6 +1094,29 @@ describe('registryManager tests', () => { expect(queued()).to.deep.equal(['DeadApp']); }); + describe('an app with no height', () => { + afterEach(() => { + globalState.testInstallingApps.clear(); + }); + + it('should leave an app being tested installed', async () => { + installed = [spec('TestedApp', 't', undefined, 100)]; + globalState.testInstallingApps.add('TestedApp'); + + await registryManager.expireGlobalApplications(); + + expect(queued()).to.deep.equal([]); + }); + + it('should uninstall one no test is running for', async () => { + installed = [spec('LeftApp', 't', undefined, 100)]; + + await registryManager.expireGlobalApplications(); + + expect(queued()).to.deep.equal(['LeftApp']); + }); + }); + it('should expire an app at its expiration height, installed or only global', async () => { // term ends at the tip await database.collection(appsInformation).insertMany([spec('EndsNowApp', 'n', 3007900, 100), spec('LiveApp', 'l', 3007900, 101)]); @@ -1203,11 +1228,12 @@ describe('registryManager tests', () => { promise.then(() => { state.done = true; }, () => { state.done = true; }); return state; }; - // the rebuild reads the scanned height from mongo before it reaches the stub + // the rebuild reads the scanned height from mongo before it reaches the + // stub, so this waits for the stub however long that read takes const rebuildsStarted = async (count) => { - for (let i = 0; i < 1000 && rebuildStub.callCount < count; i += 1) { + while (rebuildStub.callCount < count) { // eslint-disable-next-line no-await-in-loop - await new Promise((resolve) => { setImmediate(resolve); }); + await new Promise((resolve) => { setTimeout(resolve, 1); }); } expect(rebuildStub.callCount).to.equal(count); }; @@ -1316,15 +1342,22 @@ describe('registryManager tests', () => { type: 'fluxappregister', hash, height: 3000000, timestamp: 1, appSpecifications: { name: `App${hash}`, version: 3, owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC' }, }); const unresolvedSignals = () => emitSpy.args.filter(([event]) => event === SYNC_EVENTS.HASH_UNRESOLVED).length; + const hashIndex = 'query for getting app hash'; beforeEach(async () => { daemonDb = db.db(config.database.daemon.database); await daemonDb.collection(appsHashes).deleteMany({}); + // the node's own index; without it every correction scans the whole collection + await daemonDb.collection(appsHashes).createIndex({ hash: 1 }, { name: hashIndex, unique: true }); await database.collection(appsMessages).deleteMany({}); sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 3000100 } }); emitSpy = sinon.spy(appSyncEvents, 'emit'); }); + afterEach(async () => { + await daemonDb.collection(appsHashes).dropIndex(hashIndex); + }); + it('should signal a hash to fetch when it marks a received message missing', async () => { await daemonDb.collection(appsHashes).insertOne(record('lost', { message: true })); @@ -1349,6 +1382,20 @@ describe('registryManager tests', () => { expect(unresolvedSignals()).to.equal(0); }); + it('reads only the message hashes and writes its corrections in batches', async () => { + const count = 5001; + await daemonDb.collection(appsHashes).insertMany(Array.from({ length: count }, (_, i) => record(`h${i}`, { message: false }))); + await database.collection(appsMessages).insertMany(Array.from({ length: count }, (_, i) => message(`h${i}`))); + const reads = sinon.spy(dbHelper, 'findInDatabase'); + const writes = sinon.spy(Collection.prototype, 'bulkWrite'); + + const { changed } = await registryManager.reconstructAppMessagesHashCollection(); + + expect(changed).to.equal(count); + expect(reads.firstCall.args[3]).to.deep.equal({ projection: { _id: 0, hash: 1 } }); + expect(writes.getCalls().map((call) => call.args[0].length)).to.deep.equal([5000, 1]); + }); + it('should signal from the operator route', async () => { await daemonDb.collection(appsHashes).insertOne(record('lost', { message: true })); sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); @@ -1393,6 +1440,32 @@ describe('registryManager tests', () => { expect(await registryManager.isGoverningAppMessage('NoSuchApp', 'second')).to.equal(false); }); + it('rejects when the row cannot be written, never reporting the app stored', async () => { + atHeight(150); + const reg = { type: 'fluxappregister', hash: 'unwritten', height: 100 }; + await logMessages('UnwrittenApp', [reg]); + sinon.stub(dbHelper, 'replaceOneInDatabase').rejects(new Error('connection timed out')); + + const failure = await promote('UnwrittenApp', reg).catch((err) => err); + + expect(failure, 'an unwritten row was reported stored').to.be.an('error'); + expect(await stored('UnwrittenApp')).to.equal(null); + }); + + it('leaves a row that already holds the message as it is', async () => { + atHeight(150); + const reg = { type: 'fluxappregister', hash: 'heldhash', height: 100 }; + await logMessages('HeldApp', [reg]); + await promote('HeldApp', reg); + const replace = sinon.spy(dbHelper, 'replaceOneInDatabase'); + const clearErrors = sinon.spy(dbHelper, 'removeDocumentsFromCollection'); + + await promote('HeldApp', reg); + + sinon.assert.notCalled(replace); + expect(clearErrors.called, 'the app\'s install errors were cleared for a write that changed nothing').to.be.false; + }); + it('should store a registration, then the update that follows it', async () => { atHeight(150); const reg = { type: 'fluxappregister', hash: 'oldhash', height: 100 }; @@ -1572,56 +1645,6 @@ describe('registryManager tests', () => { }); }); - describe('getAllGlobalApplications tests', () => { - beforeEach(async () => { - const collection = config.database.appsglobal.collections.appsInformation; - const apps = [ - { - name: 'App1', - owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', - height: 100, - }, - { - name: 'App2', - owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', - height: 200, - }, - ]; - - try { - await database.collection(collection).drop(); - } catch (err) { - // Collection doesn't exist - } - await dbHelper.insertManyToDatabase(database, collection, apps); - }); - - it('should return all global applications', async () => { - const result = await registryManager.getAllGlobalApplications(); - - expect(result).to.be.an('array'); - expect(result.length).to.be.at.least(2); - }); - - it('should return applications with specific projections', async () => { - const result = await registryManager.getAllGlobalApplications(['name', 'owner']); - - expect(result).to.be.an('array'); - if (result.length > 0) { - expect(result[0]).to.have.property('name'); - expect(result[0]).to.have.property('owner'); - } - }); - - it('should sort applications by height', async () => { - const result = await registryManager.getAllGlobalApplications(['name', 'height']); - - if (result.length > 1) { - expect(result[0].height).to.be.at.most(result[1].height); - } - }); - }); - describe('getInstalledApps tests', () => { it('should return installed apps from local database', async () => { const result = await registryManager.getInstalledApps(); @@ -1629,21 +1652,13 @@ describe('registryManager tests', () => { expect(result).to.be.an('array'); }); - it('should handle errors and return empty array', async () => { + it('should reject when the read fails, never answer an empty list', async () => { sinon.stub(dbHelper, 'findInDatabase').rejects(new Error('Database error')); - const result = await registryManager.getInstalledApps(); + const error = await registryManager.getInstalledApps().catch((err) => err); - expect(result).to.be.an('array'); - expect(result).to.be.empty; - }); - }); - - describe('getRunningApps tests', () => { - it('should return running apps from global locations', async () => { - const result = await registryManager.getRunningApps(); - - expect(result).to.be.an('array'); + expect(error).to.be.instanceOf(Error); + expect(error.message).to.equal('Database error'); }); }); @@ -1666,6 +1681,21 @@ describe('registryManager tests', () => { ip: '192.168.1.2:16127', hash: 'hash3', }, + { + name: 'BeginsWithTheIp', + ip: '192.168.1.10:16127', + hash: 'hash4', + }, + { + name: 'MatchesOnlyAsAPattern', + ip: '192x168x1x1:16127', + hash: 'hash5', + }, + { + name: 'OtherPort', + ip: '192.168.1.1:16137', + hash: 'hash6', + }, ]; try { @@ -1676,14 +1706,29 @@ describe('registryManager tests', () => { await dbHelper.insertManyToDatabase(database, collection, testLocations); }); - it('should return apps running on specific IP', async () => { + it('should return apps running on specific IP, at every port', async () => { const result = await registryManager.getRunningAppIpList('192.168.1.1'); - expect(result).to.be.an('array'); - expect(result.length).to.equal(2); - result.forEach((app) => { - expect(app.ip).to.include('192.168.1.1'); - }); + expect(result.map((app) => app.name).sort()).to.deep.equal(['App1', 'App2', 'OtherPort']); + }); + + // An address that only begins with the IP, or matches it with its dots read + // as a pattern, is another node. + it('returns no app at an address that only begins with the IP or matches it as a pattern', async () => { + const names = (await registryManager.getRunningAppIpList('192.168.1.1')).map((app) => app.name); + + expect(names).to.not.include('BeginsWithTheIp'); + expect(names).to.not.include('MatchesOnlyAsAPattern'); + }); + + it('reads a socket address as its IP', async () => { + const result = await registryManager.getRunningAppIpList('192.168.1.1:16137'); + + expect(result.map((app) => app.name).sort()).to.deep.equal(['App1', 'App2', 'OtherPort']); + }); + + it('returns nothing for no address', async () => { + expect(await registryManager.getRunningAppIpList('')).to.deep.equal([]); }); it('should return empty array for IP with no apps', async () => { @@ -2027,3 +2072,51 @@ describe('registryManager tests', () => { }); }); }); + +describe('registryManager registerAppGlobalyApi tests', () => { + const spec = { + version: 8, + name: 'registertest', + description: 'register test', + owner: '1CbErtneaX2QVyUfwU7JGB7VzvPgrgc3uC', + enterprise: 'blob', + }; + + const register = (appSpecification) => new Promise((resolve) => { + const req = new EventEmitter(); + const res = { json: (response) => resolve(response) }; + registryManager.registerAppGlobalyApi(req, res); + req.emit('data', JSON.stringify({ + appSpecification, timestamp: Date.now(), signature: 'sig', type: 'fluxappregister', version: 1, + })); + req.emit('end'); + }); + + beforeEach(() => { + sinon.stub(verificationHelper, 'verifyPrivilege').resolves(true); + sinon.stub(peerManager, 'outboundCount').get(() => 100); + sinon.stub(peerManager, 'inboundCount').get(() => 100); + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 2500000 } }); + }); + + afterEach(() => { + sinon.restore(); + }); + + [ + ['an invalid app name', { name: 'register_test' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should refuse an enterprise spec with ${description}`, async () => { + const response = await register({ ...spec, ...change }); + expect(response.status).to.equal('error'); + expect(response.data.message).to.equal('Invalid app name or owner'); + }); + }); + + it('should decrypt a spec with a valid app name and owner', async () => { + // Off ArcaneOS the decrypt refuses, which shows the spec reached it. + const response = await register(spec); + expect(response.data.message).to.equal('Application Specifications can only be validated on a node running Arcane OS.'); + }); +}); diff --git a/tests/unit/registryRebuild.test.js b/tests/unit/registryRebuild.test.js index d8e55ac7e2..a649592f3e 100644 --- a/tests/unit/registryRebuild.test.js +++ b/tests/unit/registryRebuild.test.js @@ -11,6 +11,8 @@ const fs = require('fs'); const path = require('path'); const { expect } = require('chai'); +const sinon = require('sinon'); +const { MongoNetworkTimeoutError } = require('mongodb'); const espree = require('espree'); const dbHelper = require('../../ZelBack/src/services/dbHelper'); @@ -276,6 +278,46 @@ describe('registry rebuild: who holds a name', () => { expect(await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb)).to.equal(true); }); + it('runs its index builds and its $group on the maintenance client', async () => { + await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); + const onMaintenance = sinon.spy(dbHelper.maintenanceConnection(), 'db'); + try { + await rebuild(); + } finally { + onMaintenance.restore(); + } + sinon.assert.calledWith(onMaintenance, globalDb.databaseName); + }); + + it('runs the reindex check\'s index build and $group on the maintenance client', async () => { + await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); + const onMaintenance = sinon.spy(dbHelper.maintenanceConnection(), 'db'); + try { + await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb); + } finally { + onMaintenance.restore(); + } + sinon.assert.calledWith(onMaintenance, globalDb.databaseName); + }); + + it('rejects when the registry cannot be read, rather than reading it as consistent', async () => { + await seed([msg({ hash: 'holder', owner: 'alice', height: 1000 })], [pay('holder', 1000, 0)]); + await rebuild(); + const unreadable = { + collection: (name) => (name === INFO + ? { aggregate: () => ({ next: async () => { throw new MongoNetworkTimeoutError('timed out'); } }) } + : globalDb.collection(name)), + }; + + const error = await dbHelper.isReindexAppsInformationRequired(unreadable, MESSAGES, INFO, HEIGHT, daemonDb).catch((caught) => caught); + + expect(error).to.be.instanceOf(MongoNetworkTimeoutError); + }); + + it('answers null, not consistent, when the messages give no live app to validate against', async () => { + expect(await dbHelper.isReindexAppsInformationRequired(globalDb, MESSAGES, INFO, HEIGHT, daemonDb)).to.equal(null); + }); + it('removes this node\'s install when the newest message is another owner\'s live registration and the holder\'s app has expired', async () => { await seed( [msg({ hash: 'holder', owner: 'alice', height: 1000, expire: 1500 }), msg({ hash: 'taker', owner: 'mallory', height: 2000 })], diff --git a/tests/unit/residentialNodeDosService.test.js b/tests/unit/residentialNodeDosService.test.js index 538cfc634d..8589b1c336 100644 --- a/tests/unit/residentialNodeDosService.test.js +++ b/tests/unit/residentialNodeDosService.test.js @@ -20,6 +20,7 @@ describe('residentialNodeDosService tests', () => { let benchmarkServiceStub; let dbHelperStub; let markerStore; + let applyUpdate; let globalStateStub; let deps; let installedApps; @@ -101,12 +102,17 @@ describe('residentialNodeDosService tests', () => { // The settle marker lives in mongo, so the double has to remember it across // a reload - surviving a restart is the whole point of that field. markerStore = new Map(); + applyUpdate = (query, update) => { + const held = { _id: query._id, ...(markerStore.get(query._id) || {}), ...update.$set }; + Object.entries(update.$max || {}).forEach(([field, value]) => { + if (held[field] === undefined || value > held[field]) held[field] = value; + }); + markerStore.set(query._id, held); + }; dbHelperStub = { databaseConnection: sinon.stub().returns({ db: sinon.stub().returns({}) }), findOneInDatabase: sinon.stub().callsFake(async (db, coll, query) => markerStore.get(query._id) || null), - findOneAndUpdateInDatabase: sinon.stub().callsFake(async (db, coll, query, update) => { - markerStore.set(query._id, { _id: query._id, ...(markerStore.get(query._id) || {}), ...update.$set }); - }), + findOneAndUpdateInDatabase: sinon.stub().callsFake(async (db, coll, query, update) => applyUpdate(query, update)), findOneAndDeleteInDatabase: sinon.stub().callsFake(async (db, coll, query) => { markerStore.delete(query._id); }), @@ -431,6 +437,28 @@ describe('residentialNodeDosService tests', () => { expect(service.isEvacuating()).to.equal(false); }); + it('counts nothing of a window a verdict flip could not clear, until it is cleared', async () => { + installedApps = [{ name: 'someapp' }]; + markerStore.set('residentialDos', windowServed()); + const residential = geolocationServiceStub.getNetworkClassification(); + dbHelperStub.findOneAndDeleteInDatabase.rejects(new Error('marker delete failed')); + geolocationServiceStub.getNetworkClassification.returns({ classification: CLASSIFICATION.DATACENTER }); + await service.enforceResidentialPolicy(deps); + expect(markerStore.has('residentialDos')).to.equal(true); + + geolocationServiceStub.getNetworkClassification.returns(residential); + await service.enforceResidentialPolicy(deps); + + expect(service.isEvacuating(), 'the window of the earlier verdict is not counted').to.equal(false); + expect(markerStore.get('residentialDos').observedMs).to.equal(service.SETTLE_MS + 1000); + + dbHelperStub.findOneAndDeleteInDatabase.callsFake(async (db, coll, query) => { markerStore.delete(query._id); }); + await service.enforceResidentialPolicy(deps); + + expect(service.isEvacuating()).to.equal(false); + expect(markerStore.get('residentialDos').observedMs, 'a fresh window').to.equal(0); + }); + it('stops evacuating when the marker cannot be read', async () => { installedApps = [{ name: 'someapp' }]; markerStore.set('residentialDos', windowServed()); @@ -505,6 +533,23 @@ describe('residentialNodeDosService tests', () => { expect(markerStore.get('residentialDos').observedMs).to.equal(service.CHECK_INTERVAL_MS); }); + it('keeps the observed time and evacuates nothing when the marker cannot be read', async () => { + const now = Date.now(); + markerStore.set('residentialDos', { + _id: 'residentialDos', + residentialSince: now - (5 * 24 * HOUR), + lastConfirmedAt: now - service.CHECK_INTERVAL_MS, + observedMs: service.SETTLE_MS - HOUR, + }); + dbHelperStub.findOneInDatabase.rejects(new Error('marker read failed')); + + await service.enforceResidentialPolicy(deps); + + sinon.assert.notCalled(dbHelperStub.findOneAndUpdateInDatabase); + expect(markerStore.get('residentialDos').observedMs).to.equal(service.SETTLE_MS - HOUR); + expect(service.isEvacuating()).to.equal(false); + }); + it('starts a marker written before observed time was kept at zero, not credited', async () => { // An in-place upgrade. The old marker records when the clock started, // which is precisely the measure being replaced, so it earns nothing - @@ -568,6 +613,65 @@ describe('residentialNodeDosService tests', () => { expect(markerStore.get('residentialDos').lastEvacuationAt).to.be.a('number'); }); + it('writes a departure time it could not record on the next tick', async () => { + installedApps = [{ name: 'someapp' }]; + markerStore.set('residentialDos', windowServed()); + dbHelperStub.findOneAndUpdateInDatabase.onFirstCall().rejects(new Error('marker write failed')); + + service.noteEvacuated('someapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(markerStore.get('residentialDos').lastEvacuationAt).to.equal(undefined); + + await service.enforceResidentialPolicy(deps); + + expect(markerStore.get('residentialDos').lastEvacuationAt).to.be.a('number'); + }); + + it('keeps the newer departure time when an older one it could not record is written again', async () => { + installedApps = [{ name: 'someapp' }]; + markerStore.set('residentialDos', windowServed()); + const wall = sinon.stub(Date, 'now'); + try { + dbHelperStub.findOneAndUpdateInDatabase.onFirstCall().rejects(new Error('marker write failed')); + wall.returns(1000000); + service.noteEvacuated('someapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + wall.returns(2000000); + service.noteEvacuated('otherapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + expect(markerStore.get('residentialDos').lastEvacuationAt, 'the newer departure was not recorded').to.equal(2000000); + + await service.enforceResidentialPolicy(deps); + } finally { + wall.restore(); + } + + expect(markerStore.get('residentialDos').lastEvacuationAt, 'an older departure replaced a newer one').to.equal(2000000); + }); + + it('keeps the newer departure time when an older write lands after it', async () => { + markerStore.set('residentialDos', windowServed()); + let landOlder; + dbHelperStub.findOneAndUpdateInDatabase.onFirstCall().callsFake(async (db, coll, query, update) => { + await new Promise((resolve) => { landOlder = resolve; }); + applyUpdate(query, update); + }); + const wall = sinon.stub(Date, 'now'); + try { + wall.returns(1000000); + service.noteEvacuated('someapp'); + wall.returns(2000000); + service.noteEvacuated('otherapp'); + await new Promise((resolve) => { setImmediate(resolve); }); + landOlder(); + await new Promise((resolve) => { setImmediate(resolve); }); + } finally { + wall.restore(); + } + + expect(markerStore.get('residentialDos').lastEvacuationAt, 'an older departure replaced a newer one').to.equal(2000000); + }); + it('gives an empty location list the longest wait, not the shortest', async () => { // `index < 0 ? ordered.length : index` made an EMPTY list position 0 - // the front of the queue - inverting the rule the sibling test asserts. An diff --git a/tests/unit/resourceQueryService.test.js b/tests/unit/resourceQueryService.test.js index 372282417c..2c427290a8 100644 --- a/tests/unit/resourceQueryService.test.js +++ b/tests/unit/resourceQueryService.test.js @@ -472,6 +472,16 @@ describe('resourceQueryService tests', () => { expect(res.json.firstCall.args[0].status).to.equal('error'); }); + it('rejects with the read\'s own error for a caller inside the node', async () => { + const failure = new Error('connection timed out'); + sinon.stub(dbHelper, 'databaseConnection').returns({ db: () => ({}) }); + sinon.stub(dbHelper, 'findInDatabase').rejects(failure); + + const outcome = await resourceQueryService.appsResources().catch((err) => err); + + expect(outcome, 'a node that cannot count its apps answered totals').to.equal(failure); + }); + it('should include filesystem overhead for each app/component', async () => { const req = {}; const res = { diff --git a/tests/unit/serviceManager.test.js b/tests/unit/serviceManager.test.js index 3dc913a157..5ba74fcc61 100644 --- a/tests/unit/serviceManager.test.js +++ b/tests/unit/serviceManager.test.js @@ -1,11 +1,3 @@ -// ensureIndex is the boot sequence's self-healing index builder. It runs in the -// awaited stretch of startFluxFunctions, before any service or interval starts, -// so a throw re-runs boot safely: the recoverable failures (an options conflict, -// or a unique build blocked by duplicate rows) are healed here, and anything -// else rethrows - a transient blip heals on the next boot pass, a genuinely -// broken DB wedges loudly. dedupeByKey is the recovery strategy a unique build -// passes when its key is the row's identity. - const chai = require('chai'); const chaiAsPromised = require('chai-as-promised'); const sinon = require('sinon'); @@ -13,266 +5,39 @@ const sinon = require('sinon'); chai.use(chaiAsPromised); const { expect } = chai; -const { - ensureIndex, ensureIndexes, dedupeByKey, dosState, -} = require('../../ZelBack/src/services/serviceManager'); +const { prepareDatabaseSchema, dosState } = require('../../ZelBack/src/services/serviceManager'); +const { untilSucceeds } = require('../../ZelBack/src/services/utils/untilSucceeds'); +const dbHelper = require('../../ZelBack/src/services/dbHelper'); +const serviceHelper = require('../../ZelBack/src/services/serviceHelper'); const fluxNetworkHelper = require('../../ZelBack/src/services/fluxNetworkHelper'); -const log = require('../../ZelBack/src/lib/log'); - -describe('serviceManager ensureIndex', () => { - let logErrorSpy; - - beforeEach(() => { - logErrorSpy = sinon.spy(log, 'error'); - }); - - afterEach(() => { - sinon.restore(); - }); - - function stubCollection(createIndex) { - return { - collectionName: 'somecollection', - createIndex, - listIndexes: sinon.stub(), - dropIndex: sinon.stub().resolves(), - }; - } - - const duplicateError = () => { - const err = new Error('E11000 duplicate key error collection'); - err.code = 11000; - err.codeName = 'DuplicateKey'; - return err; - }; - - it('builds the index and touches nothing else on success', async () => { - const createIndex = sinon.stub().resolves('spec_1'); - const collection = stubCollection(createIndex); - - await ensureIndex(collection, { spec: 1 }, { unique: true }); - - sinon.assert.calledOnceWithExactly(createIndex, { spec: 1 }, { unique: true }); - sinon.assert.notCalled(collection.dropIndex); - sinon.assert.notCalled(logErrorSpy); - }); - - it('drops a conflicting index by its actual name and recreates', async () => { - const conflict = new Error('Index with name: spec_1 already exists with different options'); - conflict.codeName = 'IndexOptionsConflict'; - const createIndex = sinon.stub(); - createIndex.onFirstCall().rejects(conflict); - createIndex.onSecondCall().resolves('spec_1'); - const collection = stubCollection(createIndex); - collection.listIndexes.returns({ - toArray: async () => [ - { key: { _id: 1 }, name: '_id_' }, - { key: { spec: 1 }, name: 'oldname' }, - ], - }); - - await ensureIndex(collection, { spec: 1 }, { expireAfterSeconds: 60 }); - - sinon.assert.calledOnceWithExactly(collection.dropIndex, 'oldname'); - sinon.assert.calledTwice(createIndex); - sinon.assert.notCalled(logErrorSpy); - }); - - it('runs the recovery strategy and rebuilds when duplicate rows block a unique index', async () => { - // The node ends up WITH the index, not running degraded without it. - const createIndex = sinon.stub(); - createIndex.onFirstCall().rejects(duplicateError()); - createIndex.onSecondCall().resolves('hash_1'); - const collection = stubCollection(createIndex); - const recover = sinon.stub().resolves(3); - - await ensureIndex(collection, { hash: 1 }, { unique: true }, recover); - - sinon.assert.calledOnceWithExactly(recover, collection, { hash: 1 }, { unique: true }); - sinon.assert.calledTwice(createIndex); - // recover ran before the rebuild - sinon.assert.callOrder(createIndex, recover, createIndex); - }); - - it('rethrows a duplicate-key failure with no recovery strategy, rather than skipping', async () => { - // A unique build with no strategy is not silently swallowed - it must be - // seen, not hidden until the next reboot. - const createIndex = sinon.stub().rejects(duplicateError()); - const collection = stubCollection(createIndex); - - await expect(ensureIndex(collection, { hash: 1 }, { unique: true })).to.be.rejectedWith('E11000'); - sinon.assert.calledOnce(createIndex); - }); - - it('rethrows a transient failure so the boot retry can heal it', async () => { - // Not swallowed: index setup runs before services start, so the retry loop - // re-runs safely and a transient blip heals rather than being skipped. - const transient = new Error('connection 5 to mongo timed out'); - transient.codeName = 'NetworkTimeout'; - const createIndex = sinon.stub().rejects(transient); - const collection = stubCollection(createIndex); - const recover = sinon.stub().resolves(0); - - await expect(ensureIndex(collection, { spec: 1 }, {}, recover)).to.be.rejectedWith('timed out'); - // recovery is only for the duplicate-key case, not for a transient error - sinon.assert.notCalled(recover); - }); - - it('rethrows if the rebuild after recovery still fails', async () => { - const createIndex = sinon.stub().rejects(duplicateError()); - const collection = stubCollection(createIndex); - const recover = sinon.stub().resolves(1); - - await expect(ensureIndex(collection, { hash: 1 }, { unique: true }, recover)).to.be.rejectedWith('E11000'); - sinon.assert.calledOnce(recover); - sinon.assert.calledTwice(createIndex); - }); -}); - -describe('serviceManager dedupeByKey', () => { - function stubCollection(groups) { - return { - collectionName: 'somecollection', - aggregate: sinon.stub().returns({ toArray: async () => groups }), - deleteMany: sinon.stub().resolves({ deletedCount: 0 }), - }; - } - - it('removes all but the newest per key group and reports the count', async () => { - // ids come newest-first from the pipeline's _id sort, so slice(1) keeps the - // newest and removes the rest. - const collection = stubCollection([ - { _id: { k0: 'a' }, ids: ['new1', 'old1', 'old2'] }, - { _id: { k0: 'b' }, ids: ['new3', 'old3'] }, - ]); - - const removed = await dedupeByKey(collection, { hash: 1 }, { unique: true }); - expect(removed).to.equal(3); - sinon.assert.calledOnceWithExactly(collection.deleteMany, { _id: { $in: ['old1', 'old2', 'old3'] } }); - }); +describe('untilSucceeds', () => { + afterEach(() => sinon.restore()); - it('builds the group key from every field of the index spec', async () => { - const collection = stubCollection([]); + it('runs a failing step again after a wait, until it succeeds, and answers what it resolved', async () => { + const wait = sinon.stub(serviceHelper, 'delay').resolves(); + const step = sinon.stub(); + step.onFirstCall().rejects(new Error('connection timed out')); + step.onSecondCall().rejects(new Error('connection timed out')); + step.resolves('prepared'); - await dedupeByKey(collection, { 'data.name': 1, 'data.ip': 1 }, {}); + expect(await untilSucceeds('a step', step)).to.equal('prepared'); - const [pipeline] = collection.aggregate.firstCall.args; - const group = pipeline.find((stage) => stage.$group); - expect(group.$group._id).to.deep.equal({ k0: '$data.name', k1: '$data.ip' }); - }); - - it('honours the index partial filter so it only touches covered rows', async () => { - const collection = stubCollection([]); - const partial = { incidentKey: { $exists: true } }; - - await dedupeByKey(collection, { appName: 1, eventType: 1, incidentKey: 1 }, { partialFilterExpression: partial }); - - const [pipeline] = collection.aggregate.firstCall.args; - expect(pipeline[0]).to.deep.equal({ $match: partial }); - }); - - it('deletes nothing when there are no duplicates', async () => { - const collection = stubCollection([]); - - const removed = await dedupeByKey(collection, { hash: 1 }, {}); - - expect(removed).to.equal(0); - sinon.assert.notCalled(collection.deleteMany); + sinon.assert.calledThrice(step); + sinon.assert.calledTwice(wait); + sinon.assert.alwaysCalledWithExactly(wait, 15 * 1000); }); }); -// ensureIndexes is the same guarantee taken in one command. mongo pays a fixed -// cost per index BUILD whatever the data size, so a node asserting its schema -// one index at a time pays it 34 times over 14 collections - and in the -// integration harness, where ten nodes share one mongod and every database is -// new, ten nodes do it at once (measured: 938 builds per fleet boot). The batch -// is the fast path only: anything that fails falls back to ensureIndex, because -// the healing above has to know WHICH index failed and a batch rejection does -// not say. -describe('serviceManager ensureIndexes', () => { - afterEach(() => { - sinon.restore(); - }); - - function stubCollection({ createIndexes, createIndex } = {}) { - return { - collectionName: 'somecollection', - createIndexes: createIndexes || sinon.stub().resolves(['a_1']), - createIndex: createIndex || sinon.stub().resolves('a_1'), - listIndexes: sinon.stub(), - dropIndex: sinon.stub().resolves(), - }; - } - - it('asserts every index in ONE command, not one command per index', async () => { - const collection = stubCollection(); - - await ensureIndexes(collection, [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'by name' }, - { key: { ip: 1, name: 1 } }, - ]); - - sinon.assert.calledOnce(collection.createIndexes); - sinon.assert.notCalled(collection.createIndex); - expect(collection.createIndexes.firstCall.args[0]).to.deep.equal([ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { name: 1 }, name: 'by name' }, - { key: { ip: 1, name: 1 } }, - ]); - }); - - it('never hands mongo the recovery strategy, which is ours', async () => { - const collection = stubCollection(); - - await ensureIndexes(collection, [ - { key: { hash: 1 }, unique: true, recover: dedupeByKey }, - ]); - - const [models] = collection.createIndexes.firstCall.args; - expect(models).to.deep.equal([{ key: { hash: 1 }, unique: true }]); - expect(models[0]).to.not.have.property('recover'); - }); - - it('falls back to one at a time when the batch is refused, and still heals', async () => { - const duplicate = new Error('E11000 duplicate key error collection'); - duplicate.code = 11000; - duplicate.codeName = 'DuplicateKey'; - - const createIndex = sinon.stub(); - createIndex.onFirstCall().resolves('expireAt_1'); - createIndex.onSecondCall().rejects(duplicate); - createIndex.onThirdCall().resolves('hash_1'); - - const collection = stubCollection({ - createIndexes: sinon.stub().rejects(duplicate), - createIndex, - }); - const recover = sinon.stub().resolves(2); - - await ensureIndexes(collection, [ - { key: { expireAt: 1 }, expireAfterSeconds: 0 }, - { key: { hash: 1 }, unique: true, recover }, - ]); - - // one per index, with the recovery run between the failed build and its retry - sinon.assert.calledThrice(createIndex); - sinon.assert.calledOnce(recover); - sinon.assert.calledWithExactly(createIndex.firstCall, { expireAt: 1 }, { expireAfterSeconds: 0 }); - sinon.assert.calledWithExactly(createIndex.secondCall, { hash: 1 }, { unique: true }); - sinon.assert.calledWithExactly(createIndex.thirdCall, { hash: 1 }, { unique: true }); - }); - it('still rethrows what the per-index path cannot heal', async () => { - const fatal = new Error('not authorized on db to execute command'); - fatal.codeName = 'Unauthorized'; +describe('serviceManager prepareDatabaseSchema', () => { + afterEach(() => sinon.restore()); - const collection = stubCollection({ - createIndexes: sinon.stub().rejects(fatal), - createIndex: sinon.stub().rejects(fatal), - }); + it('rejects when the login rows cannot be cleared of signatures, so boot prepares the schema again', async () => { + sinon.stub(dbHelper, 'maintenanceConnection').returns({ db: (name) => ({ name, client: 'maintenance' }) }); + const unset = sinon.stub(dbHelper, 'updateInDatabase').rejects(new Error('connection timed out')); - await expect(ensureIndexes(collection, [{ key: { a: 1 } }])).to.be.rejectedWith('not authorized'); + await expect(prepareDatabaseSchema()).to.be.rejectedWith('connection timed out'); + expect(unset.firstCall.args[0].client, 'the schema runs on the maintenance client').to.equal('maintenance'); }); }); diff --git a/tests/unit/syncthingFolderStateMachine.test.js b/tests/unit/syncthingFolderStateMachine.test.js index 533abac319..cbaf0a8423 100644 --- a/tests/unit/syncthingFolderStateMachine.test.js +++ b/tests/unit/syncthingFolderStateMachine.test.js @@ -14,6 +14,8 @@ const syncthingService = require('../../ZelBack/src/services/syncthingService'); const syncthingServiceMock = { getDbStatus: sinon.stub(), eachDbLocalChanged: sinon.stub(), + getDbLocalChanged: sinon.stub(), + getDbFile: sinon.stub(), systemRestart: sinon.stub(), getConfig: sinon.stub(), getDbCompletion: sinon.stub(), @@ -22,6 +24,7 @@ const syncthingServiceMock = { // reports 100% complete. getDeviceId: sinon.stub().resolves('LOCAL-DEVICE'), getConfigDevices: sinon.stub(), + getConfigFolders: sinon.stub(), dbRevert: sinon.stub(), scanFolder: sinon.stub(), getDeviceStats: sinon.stub(), @@ -173,7 +176,20 @@ const peerComponentMock = { componentStateOnOtherHolders: sinon.stub(), }; +const localChangeCheck = proxyquire('../../ZelBack/src/services/appMonitoring/localChangeCheck', { + '../syncthingService': syncthingServiceMock, +}); + +// Lets every check the passes asked for run to its end. +async function checksSettled() { + for (let i = 0; i < 50; i += 1) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => { setImmediate(resolve); }); + } +} + const stateMachine = proxyquire('../../ZelBack/src/services/appMonitoring/syncthingFolderStateMachine', { + './localChangeCheck': localChangeCheck, '../dockerService': dockerServiceMock, '../syncthingService': syncthingServiceMock, '../serviceHelper': serviceHelperMock, @@ -191,6 +207,7 @@ const stateMachine = proxyquire('../../ZelBack/src/services/appMonitoring/syncth describe('syncthingFolderStateMachine tests', () => { beforeEach(() => { + globalStateMock.folderStates.clear(); peerIdentityMock.reset(); primaryRoleMock.holdAsStandby.resetHistory(); peerComponentMock.componentStateOnPeers.reset(); @@ -214,6 +231,13 @@ describe('syncthingFolderStateMachine tests', () => { globalStateMock.syncthingDevicesIDCache.clear(); syncthingServiceMock.dbRevert.reset(); syncthingServiceMock.dbRevert.resolves({ status: 'success' }); + // default: one local change, whose cluster version another device holds + syncthingServiceMock.getDbLocalChanged.reset(); + syncthingServiceMock.getDbLocalChanged.resolves({ files: [{ name: 'appdata/changed' }] }); + syncthingServiceMock.getConfigFolders.reset(); + syncthingServiceMock.getConfigFolders.resolves({ id: 'test-app', devices: [{ deviceID: 'LOCAL-DEVICE' }, { deviceID: 'PRIMARY-DEVICE' }] }); + syncthingServiceMock.getDbFile.reset(); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 }, local: { localFlags: 8 } }); syncthingServiceMock.scanFolder.reset(); syncthingServiceMock.scanFolder.resolves({}); // default: this node's syncthing has never been connected to any peer @@ -896,6 +920,26 @@ describe('syncthingFolderStateMachine tests', () => { sinon.assert.notCalled(appReconcilerMock.setControllerDesired); }); + // A ready standby is not measured, so a peer about to seed must not wait on it: + // every state but a measurement it already holds becomes no claim. + [ + ['not looked at', { state: 'notLookedAt' }, { state: 'noClaim' }], + ['held out', { state: 'heldOut' }, { state: 'noClaim' }], + ['measured', { state: 'measured', bytes: 7, newestModified: 3 }, { state: 'measured', bytes: 7, newestModified: 3 }], + ].forEach(([what, before, after]) => { + it(`records a ready single-writer standby it found ${what} as ${after.state}`, async () => { + mockParams.containerDataFlags = 'g'; + mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); + dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); + globalStateMock.folderStates.set('test-app', before); + + await stateMachine.manageFolderSyncState(mockParams); + + expect(globalStateMock.folderStates.get('test-app')).to.deep.equal(after); + }); + }); + // A receiveonly folder keeps its own deviations, owner and mode included, and // the primary counts every one of them as outstanding for this node. it('reverts a ready single-writer standby whose directories differ from the cluster', async () => { @@ -913,11 +957,17 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { + // The primary runs the app on another holder. + peerComponentMock.componentStateOnOtherHolders.resolves('running'); + // The first pass asks for the check; the next asks for the revert its answer allows. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder, 'a g: folder\'s type is its primary role\'s').to.not.have.property('type'); - expect(result.cache).to.deep.equal({ restarted: true }); + expect(result.cache).to.include({ restarted: true }); sinon.assert.calledWithExactly(publish, 'syncthing:localChangesReverted', { folder: 'test-app', files: 0, directories: 1 }); } finally { publish.restore(); @@ -940,7 +990,13 @@ describe('syncthingFolderStateMachine tests', () => { const publish = sinon.stub(fluxEventBus, 'publish'); try { + // The primary runs the app on another holder. + peerComponentMock.componentStateOnOtherHolders.resolves('running'); + // The first pass asks for the check; the next asks for the revert its answer allows. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnce(syncthingServiceMock.dbRevert); sinon.assert.neverCalledWith(publish, 'syncthing:localChangesReverted'); @@ -1047,9 +1103,23 @@ describe('syncthingFolderStateMachine tests', () => { const [readLocations, localSocketAddr, ctx, options] = peerComponentMock.componentStateOnOtherHolders.firstCall.args; expect(readLocations).to.equal(mockParams.readAppLocation); expect(localSocketAddr).to.equal(mockParams.localSocketAddr); - expect(ctx).to.include({ appId: 'test-app', identifier: 'app_test', appName: 'test-app' }); + expect(ctx).to.include({ identifier: 'app_test', appName: 'test-app' }); + expect(ctx.appIds).to.deep.equal(['test-app']); expect(options, 'a node FDM names').to.equal(undefined); }); + + // A holder running any g: component of the app is its primary, and this + // folder's unsent changes are not to go out beside it. + it('asks about every g: component of the app', async () => { + mockParams.readAppLocation = sinon.stub().resolves([]); + mockParams.electedSiblings = ['fluxweb_test-app']; + + const { othersHold } = await demote(); + await othersHold(); + + const [, , ctx] = peerComponentMock.componentStateOnOtherHolders.firstCall.args; + expect(ctx.appIds).to.deep.equal(['test-app', 'fluxweb_test-app']); + }); }); it('leaves the pause of a sending single-writer folder out of the write when what it holds cannot be read', async () => { @@ -1137,17 +1207,16 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); - expect(globalStateMock.folderHoldings.get('test-app').bytes).to.equal(5821604997); + expect(globalStateMock.folderStates.get('test-app')).to.include({ state: 'measured', bytes: 5821604997 }); }); // A read it cannot make is not a smaller claim, it is no claim - and the node // stands down on exactly that. Left up, the last good figure makes every peer - // defer to a node that is refusing to lead, and nobody seeds. - it('withdraws the published claim when it can no longer read what it holds', async () => { - globalStateMock.folderHoldings = new Map([ - ['test-app', { bytes: 5821604997, newestModified: 200 }], - ['other-app', { bytes: 900, newestModified: 100 }], - ]); + // defer to a node that is refusing to lead, and nobody seeds. Recorded as + // no claim rather than removed, so the folder is not one it has yet to look at. + it('records the folder as no claim when it can no longer read what it holds', async () => { + globalStateMock.folderStates.set('test-app', { state: 'measured', bytes: 5821604997, newestModified: 200 }); + globalStateMock.folderStates.set('other-app', { state: 'measured', bytes: 900, newestModified: 100 }); fsMock.promises.readdir.resolves([]); mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1, leaderStreak: 5, @@ -1160,8 +1229,8 @@ describe('syncthingFolderStateMachine tests', () => { await stateMachine.manageFolderSyncState(mockParams); - expect(globalStateMock.folderHoldings.has('test-app'), 'an unreadable folder still claims its last good figure').to.equal(false); - expect(globalStateMock.folderHoldings.has('other-app'), 'another folder\'s claim is not this folder\'s to withdraw').to.equal(true); + expect(globalStateMock.folderStates.get('test-app'), 'an unreadable folder still claims its last good figure').to.deep.equal({ state: 'noClaim' }); + expect(globalStateMock.folderStates.get('other-app'), 'another folder\'s claim is not this folder\'s to withdraw').to.include({ state: 'measured' }); }); // The status read for the decision is the one the disk is judged against. Read @@ -2280,7 +2349,11 @@ describe('syncthingFolderStateMachine tests', () => { receiveOnlyChangedBytes: 555, }); + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder.type).to.equal('receiveonly'); @@ -2288,6 +2361,370 @@ describe('syncthingFolderStateMachine tests', () => { sinon.assert.neverCalledWith(appReconcilerMock.setControllerDesired, sinon.match.any, 'running'); }); + // A change whose cluster version is this node's own is held by no other device: + // syncthing reverts it by deleting it. One to data another device wrote is the + // only copy of that data, so it is kept and promoted with the rest; one this node + // made itself is reverted, as any pollution is. The check runs off the pass, and + // its answer is acted on only while the folder is the one it was checked against. + describe('a local change no other device holds', () => { + const FROM_PRIMARY = ['PRIMARY:5', 'LOCAL-D:9']; + const changes = (...files) => syncthingServiceMock.getDbLocalChanged.resolves({ files }); + const onlyHere = (name) => syncthingServiceMock.getDbFile.withArgs('test-app', name).resolves({ global: { localFlags: 8 }, local: { localFlags: 8 } }); + const heldElsewhere = (name) => syncthingServiceMock.getDbFile.withArgs('test-app', name).resolves({ global: { localFlags: 0 }, local: { localFlags: 8 } }); + const status = (over = {}) => syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 1000, inSyncBytes: 1000, state: 'idle', receiveOnlyChangedFiles: 0, receiveOnlyChangedDirectories: 1, + sequence: 10, remoteSequence: { 'PRIMARY-DEVICE': 5 }, ...over, + }); + const sharedWith = (...devices) => syncthingServiceMock.getConfigFolders.resolves({ id: 'test-app', devices: devices.map((deviceID) => ({ deviceID })) }); + const pollutedCandidate = () => { + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: false, numberOfExecutions: 1 }); + mockParams.appLocation.resolves([ + { ip: '10.0.0.0:16127', runningSince: null, broadcastedAt: 1000 }, + { ip: '10.0.0.1:16127', runningSince: null, broadcastedAt: 1000 }, + ]); + status(); + }; + const readyStandby = () => { + mockParams.containerDataFlags = 'g'; + mockParams.syncFolder = { id: 'test-app', type: 'receiveonly' }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { restarted: true }); + dockerServiceMock.dockerContainerInspect.resolves({ State: { Running: false } }); + status(); + }; + const pass = () => stateMachine.manageFolderSyncState(mockParams); + const checkedPass = async () => { + await pass(); + await checksSettled(); + const result = await pass(); + await checksSettled(); + return result; + }; + + it('is neither reverted nor promoted before the check has answered', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + + const result = await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + }); + + it('is kept, and the folder is promoted', async () => { + pollutedCandidate(); + changes({ name: 'appdata/only-here', version: FROM_PRIMARY }); + onlyHere('appdata/only-here'); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted, 'a change only this node holds kept the folder from promoting').to.be.true; + }); + + it('is reverted on an r: folder when this node made it, asking nothing about it', async () => { + pollutedCandidate(); + changes({ name: 'appdata/made-here', version: ['LOCAL-D:9'] }); + + const result = await checkedPass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + sinon.assert.notCalled(syncthingServiceMock.getDbFile); + expect(result.cache.restarted).to.be.false; + }); + + it('is reverted when another device holds the cluster\'s version', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + + await checkedPass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + + it('is found behind changes other devices hold', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }, { name: 'appdata/only-here', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + onlyHere('appdata/only-here'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('is found on a later page', async () => { + pollutedCandidate(); + const held = Array.from({ length: 1000 }, (_, k) => ({ name: `appdata/held-${k}`, version: FROM_PRIMARY })); + syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 1).resolves({ files: held }); + syncthingServiceMock.getDbLocalChanged.withArgs('test-app', 2).resolves({ files: [{ name: 'appdata/only-here', version: FROM_PRIMARY }] }); + syncthingServiceMock.getDbFile.resolves({ global: { localFlags: 0 } }); + onlyHere('appdata/only-here'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + // The primary dies and is dropped from the folder: a change it held is now the + // only copy. Neither counter moves when a device leaves, so the device list is + // what makes the answer from before stale. + it('is not reverted on an answer from before the primary left the folder', async () => { + pollutedCandidate(); + changes({ name: 'appdata/save', version: FROM_PRIMARY }); + heldElsewhere('appdata/save'); + await pass(); + await checksSettled(); + sharedWith('LOCAL-DEVICE'); + onlyHere('appdata/save'); + + await pass(); + await checksSettled(); + const result = await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted, 'the only copy, kept, is promoted').to.be.true; + }); + + // A pull moves the folder's sequences and not its counts of this node's changes, + // so the answer stands while the primary keeps writing. + it('reverts on its answer while the folder only pulls, its sequences moving every pass', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + status({ sequence: 11, remoteSequence: { 'PRIMARY-DEVICE': 6 } }); + + await pass(); + await checksSettled(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + + [ + ['another change on this node', { receiveOnlyChangedFiles: 1 }], + ['a change of what this node changed', { receiveOnlyChangedBytes: 4096 }], + ].forEach(([what, moved]) => { + it(`is checked again after ${what}, and not reverted on the answer from before it`, async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + status(moved); + + await pass(); + await checksSettled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); + }); + }); + + // The counts can stay the same while the changes do not: the revert reads them + // again, and goes ahead only on the ones the check examined. + it('does not revert changes other than the ones checked, and checks them before any revert', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + changes({ name: 'appdata/other', version: FROM_PRIMARY }); + onlyHere('appdata/other'); + + await pass(); + await checksSettled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(mockParams.receiveOnlySyncthingAppsCache.get('test-app'), 'the answer the revert did not rest on').to.not.have.property('localChangeCheck'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(mockParams.receiveOnlySyncthingAppsCache.get('test-app').localChangeCheck.onlyCopy).to.equal('appdata/other'); + }); + + it('does not revert when its changes cannot be read again', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + await checksSettled(); + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); + + await pass(); + await checksSettled(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('leaves no answer on a folder whose entry was replaced while it was checked', async () => { + pollutedCandidate(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + await pass(); + const checked = mockParams.receiveOnlySyncthingAppsCache.get('test-app'); + const replacement = { restarted: false, numberOfExecutions: 0 }; + mockParams.receiveOnlySyncthingAppsCache.set('test-app', replacement); + await checksSettled(); + + expect(checked, 'the replaced entry took the answer').to.not.have.property('localChangeCheck'); + expect(replacement).to.not.have.property('localChangeCheck'); + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + }); + + it('is neither reverted nor promoted while the changes cannot be read, and is asked about again', async () => { + pollutedCandidate(); + syncthingServiceMock.getDbLocalChanged.rejects(new Error('connect ECONNREFUSED')); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + sinon.assert.calledTwice(syncthingServiceMock.getDbLocalChanged); + }); + + it('is neither reverted nor promoted while which devices share the folder cannot be read', async () => { + pollutedCandidate(); + syncthingServiceMock.getConfigFolders.rejects(new Error('connect ECONNREFUSED')); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.notCalled(syncthingServiceMock.getDbLocalChanged); + expect(result.cache.restarted).to.be.false; + }); + + // A change only this node made, on a single-writer folder, is the app's data + // while no other holder runs the app - its own writes as an earlier primary + // included - and junk beside a holder that runs it. + describe('made on this node, on a single-writer folder', () => { + const MADE_HERE = ['LOCAL-D:9']; + const othersRun = (state) => peerComponentMock.componentStateOnOtherHolders.resolves(state); + + it('is reverted while another holder runs the app', async () => { + readyStandby(); + changes({ name: 'appdata/stray', version: MADE_HERE }); + othersRun('running'); + + await checkedPass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + + it('is kept while no other holder runs the app, and said once', async () => { + readyStandby(); + changes({ name: 'appdata/written-as-primary', version: MADE_HERE }, { name: 'appdata', version: MADE_HERE }); + othersRun('notRunning'); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + await checkedPass(); + await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.calledOnceWithExactly( + publish.withArgs('syncthing:localChangesKept'), + 'syncthing:localChangesKept', + { + folder: 'test-app', files: 0, directories: 1, reason: 'noHolderRunsTheApp', + }, + ); + } finally { + publish.restore(); + } + }); + + it('is neither reverted nor promoted while whether another holder runs the app cannot be said', async () => { + pollutedCandidate(); + mockParams.containerDataFlags = 'g'; + changes({ name: 'appdata/stray', version: MADE_HERE }); + othersRun('unknown'); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.false; + }); + + // The sole survivor of an app it was once the primary of: its folder becomes + // ready to take over with its changes, rather than being held for a revert + // that deletes them. + it('lets a folder become ready to take over while no other holder runs the app', async () => { + pollutedCandidate(); + mockParams.containerDataFlags = 'g'; + changes({ name: 'appdata/written-as-primary', version: MADE_HERE }); + othersRun('notRunning'); + + const result = await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + expect(result.cache.restarted).to.be.true; + }); + + it('is reverted once a holder runs the app, though it was kept while none did', async () => { + readyStandby(); + changes({ name: 'appdata/stray', version: MADE_HERE }); + othersRun('notRunning'); + await checkedPass(); + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + + othersRun('running'); + await pass(); + await checksSettled(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + }); + + it('is kept beside a holder that runs the app when another change is the only copy of another device\'s data', async () => { + readyStandby(); + changes({ name: 'appdata/stray', version: MADE_HERE }, { name: 'appdata/only-here', version: FROM_PRIMARY }); + onlyHere('appdata/only-here'); + othersRun('running'); + + await checkedPass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.notCalled(peerComponentMock.componentStateOnOtherHolders); + }); + + it('does not ask whether another holder runs the app when every change names another device', async () => { + readyStandby(); + changes({ name: 'appdata/held', version: FROM_PRIMARY }); + heldElsewhere('appdata/held'); + + await checkedPass(); + + sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); + sinon.assert.notCalled(peerComponentMock.componentStateOnOtherHolders); + }); + }); + + it('is kept on a ready standby, said once, and costs nothing on the passes after', async () => { + readyStandby(); + changes({ name: 'appdata/only-here', version: FROM_PRIMARY }); + onlyHere('appdata/only-here'); + const publish = sinon.stub(fluxEventBus, 'publish'); + try { + await checkedPass(); + await pass(); + await pass(); + + sinon.assert.notCalled(syncthingServiceMock.dbRevert); + sinon.assert.calledOnceWithExactly( + publish.withArgs('syncthing:localChangesKept'), + 'syncthing:localChangesKept', + { folder: 'test-app', files: 0, directories: 1, reason: 'onlyCopy' }, + ); + sinon.assert.calledOnce(syncthingServiceMock.getDbLocalChanged); + sinon.assert.calledOnce(syncthingServiceMock.getDbFile); + } finally { + publish.restore(); + } + }); + }); + // Three properties of what counts as "holding", each able to hand the seed to the // wrong node. In all three this node's address sorts FIRST, so the address order // would give it the seed; only the holdings decide otherwise. @@ -2307,6 +2744,196 @@ describe('syncthingFolderStateMachine tests', () => { }); }; + // A holder that has not yet measured what it holds of the folder - an install its + // monitor has not reached - would read as holding nothing, and the field would fall + // to the address order over a holder that may hold the owner's data. So nobody + // seeds while one has not looked. This node's address sorts first and it holds + // nothing, so the address order would give it the seed. + describe('a holder that has not yet looked at the folder', () => { + const coldStartBeside = (peerAnswer) => { + mockParams.localSocketAddr = '10.0.0.1:16127'; + mockParams.appLocation.resolves([ + { ip: '10.0.0.1:16127', runningSince: 2000, broadcastedAt: 1000 }, + { ip: '10.0.0.2:16127', runningSince: 2000, broadcastedAt: 1000 }, + ]); + syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [] })); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 0, inSyncBytes: 0, state: 'idle', receiveOnlyChangedFiles: 0, + }); + axiosMock.get.resolves({ data: { data: { ready: true, folders: [], holding: {}, ...peerAnswer } } }); + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { + restarted: false, numberOfExecutions: 1, leaderStreak: 5, + }); + }; + + it('keeps any node from seeding until it has', async () => { + const count = sinon.spy(fluxEventBus, 'count'); + try { + coldStartBeside({ notLookedAt: ['test-app'] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'seeded over a holder that had not looked at what it holds').to.equal('receiveonly'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderNotLookedAt'); + } finally { + count.restore(); + } + }); + + it('a canary: the same holder, having looked and holding nothing, leaves the seed to the address order', async () => { + coldStartBeside({ notLookedAt: [] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + + it('is waited on for one of the app\'s other elected folders too', async () => { + coldStartBeside({ notLookedAt: ['sibling-app'] }); + mockParams.electedSiblings = ['sibling-app']; + // no node writes the app's other folder yet, so the election decides this one + appQueryServiceMock.holdsComponent.resolves(false); + globalStateMock.folderStates.set('sibling-app', { state: 'measured', bytes: 0, newestModified: 0 }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('receiveonly'); + }); + + // A peer too old to say reads as having looked at everything, as it always has. + it('is not assumed of a peer that does not say', async () => { + coldStartBeside({}); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + }); + + // A holder whose volume will not mount cannot seed, and the address order would + // give it the seed whenever it sorts first. Here it does: 10.0.0.1, beside this + // node at 10.0.0.2. + describe('a holder that cannot mount a volume', () => { + const BROKEN = '10.0.0.1:16127'; + const coldStartBesideBroken = (brokenAnswer, { holdsData = false, others = {} } = {}) => { + mockParams.localSocketAddr = '10.0.0.2:16127'; + mockParams.appLocation.resolves([ + { ip: BROKEN, runningSince: 2000, broadcastedAt: 1000 }, + { ip: '10.0.0.2:16127', runningSince: 2000, broadcastedAt: 1000 }, + ...Object.keys(others).map((ip) => ({ ip, runningSince: 2000, broadcastedAt: 1000 })), + ]); + syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ + files: holdsData ? [localEntry('world.dat', 5821604997, '2026-09-16T10:00:00Z')] : [], + })); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 0, inSyncBytes: 0, state: 'idle', receiveOnlyChangedFiles: holdsData ? 2 : 0, + }); + axiosMock.get.callsFake(async (url) => { + const other = Object.keys(others).find((ip) => url.includes(ip.split(':')[0])); + const answer = url.includes(BROKEN.split(':')[0]) ? brokenAnswer : (others[other] ?? {}); + return { data: { data: { ready: true, folders: [], holding: {}, ...answer } } }; + }); + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { + restarted: false, numberOfExecutions: 1, leaderStreak: 5, + }); + }; + const counting = async (run) => { + const count = sinon.spy(fluxEventBus, 'count'); + try { + return await run(count); + } finally { + count.restore(); + } + }; + + it('seeds from the holder with data, leaving out one whose volume image is on it and will not mount', async () => { + await counting(async (count) => { + coldStartBesideBroken({ heldOut: ['test-app'] }, { holdsData: true }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'the holder with data did not seed').to.equal('sendreceive'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderCannotSeed'); + }); + }); + + it('seeds nothing while a holder whose volume image is on it will not mount and no other holder has data', async () => { + await counting(async (count) => { + coldStartBesideBroken({ heldOut: ['test-app'] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'an empty copy seeded over a volume that may hold the app\'s data').to.equal('receiveonly'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderVolumeHeldOut'); + }); + }); + + it('seeds an empty copy beside a holder whose volume image is gone', async () => { + await counting(async (count) => { + coldStartBesideBroken({ volumeGone: ['test-app'] }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'the address order gave the seed to a holder with no volume').to.equal('sendreceive'); + sinon.assert.calledWith(count, 'syncthing:seedDecision', 'test-app', 'holderCannotSeed'); + sinon.assert.neverCalledWith(count, 'syncthing:seedDecision', 'test-app', 'holderVolumeHeldOut'); + }); + }); + + // A canary for the three above: the same holder, mounted and holding nothing, + // keeps the seed by the address order. + it('leaves the seed to the address order when that holder says neither', async () => { + coldStartBesideBroken({}); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('receiveonly'); + }); + + it('leaves out a holder that cannot mount the volume of one of the app\'s other elected folders', async () => { + coldStartBesideBroken({ volumeGone: ['sibling-app'] }); + mockParams.electedSiblings = ['sibling-app']; + appQueryServiceMock.holdsComponent.resolves(false); + globalStateMock.folderStates.set('sibling-app', { state: 'measured', bytes: 0, newestModified: 0 }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + + it('seeds from the holder with data among the rest, not from the next empty address', async () => { + // 10.0.0.1 cannot mount; 10.0.0.2 (this node) holds data; 10.0.0.3 holds nothing. + coldStartBesideBroken({ heldOut: ['test-app'] }, { + holdsData: true, + others: { '10.0.0.3:16127': { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }, + }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('sendreceive'); + }); + + it('leaves the seed to another holder\'s larger copy among the rest', async () => { + coldStartBesideBroken({ heldOut: ['test-app'] }, { + others: { '10.0.0.3:16127': { holding: { 'test-app': { bytes: 900, newestModified: 100 } } } }, + }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type, 'an empty copy seeded beside a holder with data').to.equal('receiveonly'); + }); + + it('seeds nothing while every other holder cannot mount', async () => { + coldStartBesideBroken({ volumeGone: ['test-app'] }, { + others: { '10.0.0.3:16127': { heldOut: ['test-app'] } }, + }); + + const result = await stateMachine.manageFolderSyncState(mockParams); + + expect(result.syncthingFolder.type).to.equal('receiveonly'); + }); + }); + // Every claim is filed under the address the election list carries, because an // object key matches exactly where addresses compare tolerantly. Keyed from // anywhere else, this node stops finding its OWN claim as soon as the two @@ -2399,6 +3026,158 @@ describe('syncthingFolderStateMachine tests', () => { // node's address sorts LAST ('9.0.0.1' after '10.0.0.2' as a string), so under the // address order it loses and the empty peer publishes an empty folder over the // customer's world. It holds the data, so it must win. + // An app with more than one g: component is seeded on one node: its primary + // runs them all, so a folder seeded anywhere else leaves no node holding every + // copy it may run on. + describe('a g: folder of an app with more than one g: component', () => { + const SIBLING = 'fluxdb_test-app'; + // A brand-new folder: nothing indexed, nothing synced, nothing held here. + const freshFolder = ({ self = '10.0.0.1:16127', peer = '10.0.0.2:16127', peerAnswer } = {}) => { + mockParams.containerDataFlags = 'g'; + mockParams.electedSiblings = [SIBLING]; + mockParams.localSocketAddr = self; + mockParams.appLocation.resolves([ + { ip: self, runningSince: 2000, broadcastedAt: 1000 }, + { ip: peer, runningSince: 2000, broadcastedAt: 1000 }, + ]); + axiosMock.get.resolves({ data: { data: { ready: true, folders: [], holding: {}, ...peerAnswer } } }); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 0, inSyncBytes: 0, state: 'idle', receiveOnlyChangedFiles: 0, + }); + mockParams.receiveOnlySyncthingAppsCache.set('test-app', { + restarted: false, numberOfExecutions: 1, leaderStreak: 5, + }); + }; + const seeds = (result) => result.cache.designationPending === true; + + afterEach(() => { + globalStateMock.folderStates.clear(); + }); + + describe('a component the spec has just added', () => { + // This node's address sorts last, so the address order seeds the peer. + const lastInOrder = { self: '9.0.0.1:16127', peer: '10.0.0.2:16127' }; + + it('is seeded by the node that holds the app\'s other g: component, wherever its address falls', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + // Canary for the test above: the same node holding none of the app leaves + // the seed to the address order. + it('is not seeded by a node holding none of the app when its address sorts last', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + // What a standby holds in the folder and never sent is a local change + // against the app's primary: the writer seeds over it, and the standby's + // copy is reverted as any standby's local change is. + it('is seeded by the writer over a standby holding bytes of this folder it never sent', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 900, newestModified: 5 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + // Canary for the test above: a peer that serves the folder is a source. + // The writer's synced copy carrying local changes of its own is reverted + // to it, never published as the seed. + it('reverts the writer\'s own local changes to a peer that serves the folder, rather than seeding them', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { holding: { 'test-app': { bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + peerHoldsTheData(); + syncthingServiceMock.getDbStatus.resolves({ + globalBytes: 4096, globalFiles: 1, inSyncBytes: 4096, state: 'idle', receiveOnlyChangedFiles: 1, + }); + + // The primary runs the app on another holder. + peerComponentMock.componentStateOnOtherHolders.resolves('running'); + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + await checksSettled(); + sinon.assert.calledWith(syncthingServiceMock.dbRevert, 'test-app'); + }); + + // A standby on a release that elects one g: component takes the empty + // folder as its seed by the address order, and never withdraws the mark. + // It never starts the component, so its folder never sends. + it('is seeded by the writer over a peer\'s seed mark for it', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { seeding: { 'test-app': { stage: 'decided', bytes: 0, newestModified: 0 } } } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + // Canary for the test above: a peer whose folder already sends has seeded + // it, and the writer stands aside. + it('is not seeded by the writer over a peer whose folder for it already sends', async () => { + freshFolder({ ...lastInOrder, peerAnswer: { folders: ['test-app'] } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(true); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + it('is left to a peer whose folder for the app\'s other g: component sends, though this node sorts first', async () => { + freshFolder({ peerAnswer: { folders: [SIBLING] } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + // Canary for the test above: with no writer anywhere, the node sorting + // first seeds. + it('is seeded by the node sorting first when no node writes the app', async () => { + freshFolder({ peerAnswer: { folders: [] } }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + }); + + describe('a cold start where the holders hold different folders', () => { + // This node holds a little of this folder; the peer holds nothing of it + // and the whole of the other one. + const split = () => { + freshFolder({ + peerAnswer: { + holding: { 'test-app': { bytes: 0, newestModified: 0 }, [SIBLING]: { bytes: 5000, newestModified: 9 } }, + }, + }); + appQueryServiceMock.holdsComponent.withArgs(SIBLING).resolves(false); + syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [localEntry('a.dat', 100)] })); + globalStateMock.folderStates.set(SIBLING, { state: 'measured', bytes: 0, newestModified: 0 }); + }; + + it('ranks the whole app, so the node holding the most of it seeds every folder', async () => { + split(); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(false); + }); + + // Canary for the test above: ranked on this folder alone, this node wins. + it('would seed this folder here ranked on this folder alone', async () => { + split(); + mockParams.electedSiblings = []; + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + + it('falls back to the address order while this node cannot say what it holds of the other folder', async () => { + split(); + globalStateMock.folderStates.delete(SIBLING); + + expect(seeds(await stateMachine.manageFolderSyncState(mockParams))).to.equal(true); + }); + }); + }); + it('the node holding the data wins the election though its address sorts last', async () => { mockParams.localSocketAddr = '9.0.0.1:16127'; syncthingServiceMock.eachDbLocalChanged.callsFake(feed({ files: [localEntry('world.sav', 5821604997, '2026-09-16T09:00:00Z')] })); @@ -2567,7 +3346,11 @@ describe('syncthingFolderStateMachine tests', () => { receiveOnlyChangedDirectories: 2, }); + // The first pass asks for the check; the next acts on its answer. + await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); const result = await stateMachine.manageFolderSyncState(mockParams); + await checksSettled(); sinon.assert.calledOnceWithExactly(syncthingServiceMock.dbRevert, 'test-app'); expect(result.syncthingFolder.type).to.equal('receiveonly'); diff --git a/tests/unit/syncthingMonitor.test.js b/tests/unit/syncthingMonitor.test.js index d985521bf8..a01d3ed9ca 100644 --- a/tests/unit/syncthingMonitor.test.js +++ b/tests/unit/syncthingMonitor.test.js @@ -5,6 +5,8 @@ const { expect } = require('chai'); const sinon = require('sinon'); const proxyquire = require('proxyquire').noCallThru(); const eventBus = require('../../ZelBack/src/services/utils/fluxEventBus'); +const folderStates = require('../../ZelBack/src/services/appMonitoring/folderStates'); +const { folderStates: folderStateMap } = require('../../ZelBack/src/services/utils/globalState'); // Create mocks for all dependencies const dbHelperMock = { @@ -36,6 +38,8 @@ const syncthingServiceMock = { systemRestart: sinon.stub().resolves(), systemResume: sinon.stub().resolves(), getDbStatus: sinon.stub(), + getDbCompletion: sinon.stub().resolves({ remoteState: 'notSharing' }), + getDeviceStats: sinon.stub().resolves({}), }; const syncthingFolderStateMachineMock = { @@ -228,6 +232,9 @@ describe('syncthingMonitor tests', () => { peerFolderLivenessMock.createPeerFolderLiveness.resetHistory(); dbHelperMock.databaseConnection.reset(); dbHelperMock.findInDatabase.reset(); + // An app's location records, read for every folder the pass configures: none. + dbHelperMock.databaseConnection.returns({ db: () => ({}) }); + dbHelperMock.findInDatabase.resolves([]); // Default stub behaviors // The node's own identity, which a real pass always has: mount safety is @@ -1433,6 +1440,177 @@ describe('syncthingMonitor tests', () => { }); }); + // What a peer about to seed reads of each receive-only folder here: it waits on one + // this node has not looked at, so every folder a pass reaches must leave that state. + describe('the folders it tells peers it has not looked at', () => { + // Folder ids already in identifier form, so the state the install records and the + // one the pass records name the same folder. + const installedApps = (apps) => mockInstalledAppsFn.resolves({ status: 'success', data: apps }); + const notLookedAt = () => folderStates.answer(new Set()).notLookedAt; + + beforeEach(() => { + folderStateMap.clear(); + }); + + afterEach(() => { + folderStateMap.clear(); + }); + + // After a restart the volumes still hold their data and nothing has been measured, + // so a peer must wait on every folder until this node has looked. + it('records every receive-only folder of an installed app as not looked at, and no other', async () => { + installedApps([ + { name: 'fluxg', version: 3, containerData: 'g:/appdata' }, + { name: 'fluxr', version: 3, containerData: 'r:/appdata' }, + { name: 'fluxs', version: 3, containerData: 's:/appdata' }, + { name: 'fluxplain', version: 3, containerData: '/appdata' }, + ]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(notLookedAt()).to.have.members(['fluxg', 'fluxr']); + }); + + it('keeps a state a folder already has', async () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + folderStateMap.set('fluxg', { state: 'measured', bytes: 10, newestModified: 1 }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'measured', bytes: 10, newestModified: 1 }); + }); + + // A volume that did not mount is never measured. Left not looked at, every peer + // would wait on this node for as long as the mount stays broken, and the app would + // start nowhere. + const volumeWillNotMount = (reason) => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingEventsConsumerMock.mountVerifyPendingIds.returns(['fluxg']); + syncthingFolderStateMachineMock.verifyFolderMountSafety.resolves({ isSafe: false, isMounted: false, reason: 'empty_unmounted_directory' }); + volumeServiceMock.ensureAppVolumeMounted.resolves({ mounted: false, reason }); + syncthingServiceMock.getConfigFolders.resolves([{ id: 'fluxg', path: '/apps/fluxg', type: 'receiveonly' }]); + syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + }; + + it('records a folder whose volume image is on the node and will not mount as held out, which no peer waits on', async () => { + volumeWillNotMount('volume_image_unrecognised'); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); + expect(notLookedAt()).to.deep.equal([]); + }); + + it('records a folder whose volume image is gone as volume gone', async () => { + volumeWillNotMount('volume_file_missing'); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'volumeGone' }); + }); + + it('keeps a folder whose volume image is gone so on a pass that finds it unmounted without asking why', async () => { + folderStateMap.set('fluxg', { state: 'volumeGone' }); + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingMonitorHelpersMock.ensureStfolderExists.resolves(false); + syncthingServiceMock.getConfigFolders.resolves([]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'volumeGone' }); + }); + + it('records a folder whose volume directory is not mounted as held out', async () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingMonitorHelpersMock.ensureStfolderExists.resolves(false); + syncthingServiceMock.getConfigFolders.resolves([]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'heldOut' }); + }); + + // A folder of an app whose spec cannot be read is never measured. Left not looked + // at, every peer would wait on this node for as long as the spec stays unreadable, + // even beside a holder that has the app's data. + const specUnreadable = (folders) => { + installedApps([{ name: 'entapp', version: 8, enterprise: 'blob' }]); + appQueryServiceMock.decryptEnterpriseApps.callsFake(async (apps) => ({ readable: [], unreadable: apps, inPlace: [] })); + syncthingServiceMock.getConfigFolders.resolves(folders); + }; + + it('records a receive-only folder of an app whose spec cannot be read as held out, which no peer waits on', async () => { + folderStateMap.set('fluxwebentapp_entapp', { state: 'notLookedAt' }); + specUnreadable([{ id: 'fluxwebentapp_entapp', path: '/apps/fluxwebentapp_entapp', type: 'receiveonly' }]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxwebentapp_entapp')).to.deep.equal({ state: 'heldOut' }); + expect(notLookedAt()).to.deep.equal([]); + }); + + it('records the receive-only folders of an app whose spec cannot be read after a restart, and no other', async () => { + specUnreadable([ + { id: 'fluxwebentapp_entapp', path: '/apps/fluxwebentapp_entapp', type: 'receiveonly' }, + { id: 'fluxdbentapp_entapp', path: '/apps/fluxdbentapp_entapp', type: 'sendreceive' }, + { id: 'fluxwebother_other', path: '/apps/fluxwebother_other', type: 'receiveonly' }, + ]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect([...folderStateMap.entries()]).to.deep.equal([['fluxwebentapp_entapp', { state: 'heldOut' }]]); + }); + + it('keeps a folder of an app whose spec cannot be read as volume gone when its volume image is gone', async () => { + folderStateMap.set('fluxwebentapp_entapp', { state: 'volumeGone' }); + specUnreadable([{ id: 'fluxwebentapp_entapp', path: '/apps/fluxwebentapp_entapp', type: 'receiveonly' }]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxwebentapp_entapp')).to.deep.equal({ state: 'volumeGone' }); + }); + + // A read that failed says nothing about what the volume holds. + const locationsUnreadable = () => { + installedApps([{ name: 'fluxg', version: 3, containerData: 'g:/appdata' }]); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingMonitorHelpersMock.ensureStfolderExists.resolves(true); + dbHelperMock.findInDatabase.rejects(new Error('database unavailable')); + syncthingServiceMock.getConfigFolders.resolves([]); + }; + + it('leaves what it measured of a folder whose locations could not be read', async () => { + folderStateMap.set('fluxg', { state: 'measured', bytes: 10, newestModified: 1 }); + locationsUnreadable(); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'measured', bytes: 10, newestModified: 1 }); + }); + + it('leaves a folder it has not looked at whose locations could not be read as not looked at, which peers wait on', async () => { + locationsUnreadable(); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + expect(folderStateMap.get('fluxg')).to.deep.equal({ state: 'notLookedAt' }); + }); + }); + describe('holds a busy app out of the config write', () => { const syncingApp = { name: 'testapp', version: 3, containerData: 'g:/appdata' }; // Bound while this file loads, which is when the module under test bound its @@ -1490,13 +1668,11 @@ describe('syncthingMonitor tests', () => { // question. Promotion answers it - everything this node holds is published // from here - so the claim stops being true as the write lands, which is // where promotedFolderIds is reconciled for the same reason. - it('drops the published holding of a folder it promotes', async () => { + it('withdraws the published holding of a folder it promotes', async () => { writesAFolder(); syncthingMonitorHelpersMock.getContainerDataFlags.returns('r'); - globalState.folderHoldings = new Map([ - ['testapp', { bytes: 5821604997, newestModified: 200 }], - ['untouchedapp', { bytes: 4096, newestModified: 100 }], - ]); + globalState.folderStates.set('testapp', { state: 'measured', bytes: 5821604997, newestModified: 200 }); + globalState.folderStates.set('untouchedapp', { state: 'measured', bytes: 4096, newestModified: 100 }); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); @@ -1506,10 +1682,10 @@ describe('syncthingMonitor tests', () => { expect(put, 'nothing was written, so this asserts nothing').to.not.equal(undefined); expect(put.args[1].find((f) => f.id === 'testapp')?.type).to.equal('sendreceive'); - expect(globalState.folderHoldings.has('testapp'), 'a promoted folder still claims receive-only holdings').to.equal(false); - // A folder this pass did not promote keeps its claim: the drop is scoped to - // what changed, not a clear of everything the node has answered for. - expect(globalState.folderHoldings.has('untouchedapp')).to.equal(true); + expect(globalState.folderStates.get('testapp'), 'a promoted folder still claims receive-only holdings').to.deep.equal({ state: 'noClaim' }); + // A folder this pass did not promote keeps its claim: the withdrawal is scoped + // to what changed, not a clear of everything the node has answered for. + expect(globalState.folderStates.get('untouchedapp')).to.include({ state: 'measured' }); }); // The election reads a folder as ready to start once it is writable, so a @@ -1578,7 +1754,9 @@ describe('syncthingMonitor tests', () => { // could not be read"; the read it is handed must not collapse the two. it('hands the state machine a location read that fails when the records cannot be read', async () => { writesAFolder(); - dbHelperMock.databaseConnection.throws(new Error('database unavailable')); + // read once for the folder's devices, then unreadable for the state machine + dbHelperMock.findInDatabase.onFirstCall().resolves([]); + dbHelperMock.findInDatabase.rejects(new Error('database unavailable')); monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); await clock.tickAsync(100); @@ -1591,6 +1769,32 @@ describe('syncthingMonitor tests', () => { expect(failure?.message).to.equal('database unavailable'); }); + // The state machine seeds an app's g: folders on one node, so each is told + // the folders of the app's other g: components; a folder of any other + // component is told none. + it('hands each g: folder the folders of the app\'s other g: components', async () => { + writesAFolder(); + mockInstalledAppsFn.resolves({ + status: 'success', + data: [{ + name: 'twin', + version: 7, + compose: [ + { name: 'db', containerData: 'g:/data' }, + { name: 'cache', containerData: 'r:/cache' }, + { name: 'web', containerData: 'g:/srv' }, + ], + }], + }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + const siblings = Object.fromEntries(syncthingFolderStateMachineMock.manageFolderSyncState.getCalls() + .map((call) => [call.args[0].appId, call.args[0].electedSiblings])); + expect(siblings).to.deep.equal({ db_twin: ['web_twin'], cache_twin: [], web_twin: ['db_twin'] }); + }); + describe('the type of a g: folder', () => { const folderCalls = () => syncthingServiceMock.adjustConfigFolders.getCalls() .filter((c) => c.args[0] === 'put' || c.args[0] === 'patch'); @@ -1813,6 +2017,111 @@ describe('syncthingMonitor tests', () => { // because a pass failed to reach the component that owns it. Deleting a live // folder drops syncthing's whole record of the app: the index, the peer // devices and, on the next pass, any mount-safety demotion standing on it. + describe('a peer no location names any more', () => { + // The peer's device is configured; `peerInFolder` is whether syncthing's + // folder still lists it, as it does until a folder write takes it out. + function folderSharedWithPeer({ peerInFolder = false } = {}) { + mockState.backupInProgress = []; + mockState.restoreInProgress = []; + syncthingMonitorHelpersMock.buildDeviceConfiguration.callsFake(async () => [{ deviceID: 'DEVICE-ID' }]); + syncthingMonitorHelpersMock.getContainerDataFlags.returns('g'); + syncthingMonitorHelpersMock.requiresSyncing.returns(true); + syncthingServiceMock.getDeviceId.resolves('DEVICE-ID'); + fluxNetworkHelperMock.getLocalSocketAddress.resolves('10.0.0.1:16127'); + mockInstalledAppsFn.resolves({ status: 'success', data: [{ name: 'testapp', version: 3, containerData: 'g:/appdata' }] }); + syncthingServiceMock.getConfigFolders.resolves([ + { id: 'testapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }, ...(peerInFolder ? [{ deviceID: 'PEER-DEVICE' }] : [])] }, + ]); + syncthingServiceMock.getConfigDevices.resolves([{ deviceID: 'PEER-DEVICE' }]); + syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + syncthingServiceMock.adjustConfigDevices.resolves({ status: 'success', data: {} }); + } + + afterEach(() => { + syncthingServiceMock.getDbCompletion.reset(); + syncthingServiceMock.getDbCompletion.resolves({ remoteState: 'notSharing' }); + syncthingServiceMock.getDeviceStats.reset(); + syncthingServiceMock.getDeviceStats.resolves({}); + }); + + // A node whose FluxOS has stopped answering stops announcing while its + // syncthing and its app carry on; once its location expires its writes reach + // no holder through this one. + it('removes its device though it is connected, once no folder lists it', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getDbCompletion.withArgs({ folder: 'testapp', device: 'PEER-DEVICE' }).resolves({ remoteState: 'valid' }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + + // Syncthing drops a deleted device from every folder that lists it, outside + // the folder's writes, which a revert of a standby's changes holds off. + it('keeps its device while a folder still lists it', async () => { + folderSharedWithPeer({ peerInFolder: true }); + syncthingServiceMock.getDbCompletion.withArgs({ folder: 'testapp', device: 'PEER-DEVICE' }).resolves({ remoteState: 'valid' }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledOnce(syncthingServiceMock.getDeviceStats); + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + + it('keeps its device while a folder this pass deletes still lists it', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getConfigFolders.resolves([ + { id: 'testapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }] }, + { id: 'goneapp', type: 'receiveonly', devices: [{ deviceID: 'DEVICE-ID' }, { deviceID: 'PEER-DEVICE' }] }, + ]); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledWithExactly(syncthingServiceMock.adjustConfigFolders, 'delete', undefined, 'goneapp'); + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + + // Its connection record is what proves it gone to a standby that remembers it as + // the app's primary. + it('keeps the entry of a device it dropped while syncthing saw it within a day', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getDeviceStats.resolves({ 'PEER-DEVICE': { lastSeen: new Date(Date.now() - 60 * 60 * 1000).toISOString() } }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + + it('removes a device seen within a day when another device holds its name', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getConfigDevices.resolves([ + { deviceID: 'PEER-DEVICE', name: '10.0.0.2:16127' }, + { deviceID: 'STALE-DEVICE', name: '10.0.0.2:16127' }, + ]); + const recent = new Date(Date.now() - 60 * 60 * 1000).toISOString(); + syncthingServiceMock.getDeviceStats.resolves({ 'PEER-DEVICE': { lastSeen: recent }, 'STALE-DEVICE': { lastSeen: recent } }); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.calledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'STALE-DEVICE'); + }); + + it('removes no device while when each was last seen cannot be read', async () => { + folderSharedWithPeer(); + syncthingServiceMock.getDeviceStats.rejects(new Error('connect ECONNREFUSED')); + + monitorControl = syncthingMonitor.syncthingApps(mockState, mockInstalledAppsFn, mockGetGlobalStateFn); + await clock.tickAsync(100); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-DEVICE'); + }); + }); + describe('unused-folder cleanup', () => { const syncingApp = { name: 'testapp', version: 3, containerData: 'g:/appdata' }; @@ -1848,6 +2157,24 @@ describe('syncthingMonitor tests', () => { sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigFolders, 'delete', undefined, 'testapp'); }); + it('leaves the folder and its peers alone when the app\'s locations cannot be read', async () => { + // Read as "no locations", the folder would be pushed shared with this node + // alone and every peer device it had removed - a primary would stop + // replicating to its standbys for as long as the read failed. + primaryMountSyncs(); + mockInstalledAppsFn.resolves({ status: 'success', data: [syncingApp] }); + dbHelperMock.findInDatabase.rejects(new Error('connection timed out')); + syncthingServiceMock.getConfigFolders.resolves([{ id: 'testapp', type: 'sendreceive', devices: [{ deviceID: 'DEVICE-ID' }, { deviceID: 'PEER-ID' }] }]); + syncthingServiceMock.getConfigDevices.resolves([{ deviceID: 'PEER-ID', name: '10.0.0.2:16127' }]); + syncthingServiceMock.adjustConfigFolders.resolves({ status: 'success', data: {} }); + + await runOnePass(); + + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigDevices, 'delete', undefined, 'PEER-ID'); + sinon.assert.neverCalledWith(syncthingServiceMock.adjustConfigFolders, 'delete', undefined, 'testapp'); + expect(syncthingFolderStateMachineMock.manageFolderSyncState.called, 'the folder was configured without its locations').to.be.false; + }); + it('keeps the folder of a component the state machine defers on first encounter', async () => { // the first-encounter deferral is a "look again next pass", not a verdict // that the folder should not exist diff --git a/tests/unit/syncthingMonitorHelpers.test.js b/tests/unit/syncthingMonitorHelpers.test.js index 37552f2639..ffe40c35c1 100644 --- a/tests/unit/syncthingMonitorHelpers.test.js +++ b/tests/unit/syncthingMonitorHelpers.test.js @@ -404,6 +404,41 @@ describe('syncthingMonitorHelpers tests', () => { expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map())).to.equal(null); }); + // A listed holder whose FluxOS has stopped answering, asked by a node whose + // FluxOS restarted and so holds nothing for it. + it('keeps the device syncthing has configured under the address of a listed peer that does not answer', async () => { + const cache = new Map(); + sandbox.stub(axios, 'get').rejects(new Error('connect ETIMEDOUT')); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', cache, 'CONFIGURED-DEVICE')).to.equal('CONFIGURED-DEVICE'); + expect(cache.has('10.0.0.1:16127'), 'a configured device is asked again on the next pass').to.equal(false); + counted('configured'); + }); + + it('keeps the configured device while a verified peer\'s syncthing has not answered and nothing is held', async () => { + verifyPeer.resolves({ + verdict: IdentityVerdict.VERIFIED, + identity: { socketAddress: '10.0.0.1:16127', pubKey: 'PUB', deviceId: null }, + }); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map(), 'CONFIGURED-DEVICE')).to.equal('CONFIGURED-DEVICE'); + counted('configured'); + }); + + it('takes a peer\'s unsigned answer over the device configured under its address', async () => { + sandbox.stub(axios, 'get').resolves({ data: { status: 'success', data: 'ANSWERED-DEVICE' } }); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map(), 'CONFIGURED-DEVICE')).to.equal('ANSWERED-DEVICE'); + counted('unsigned'); + }); + + it('configures no device for an address another node answered, whatever is configured under it', async () => { + verifyPeer.resolves({ verdict: IdentityVerdict.MISROUTED, answeredAs: '10.0.0.9:16147' }); + + expect(await helpers.getDeviceIDCached('10.0.0.1:16127', new Map(), 'CONFIGURED-DEVICE')).to.equal(null); + counted('withheld'); + }); + // The palworld case: HK held its neighbour's device under UAE's name for five // days, and the folder never synced. it('configures no device for an address another node answered, and forgets the one it held', async () => { @@ -487,6 +522,38 @@ describe('syncthingMonitorHelpers tests', () => { expect(devicesIds).to.deep.equal(['REAL-DEVICE']); }); + describe('a listed peer that gives no device', () => { + beforeEach(() => { + peerIdentityService.verifyPeer.resolves({ verdict: IdentityVerdict.UNVERIFIABLE, reason: 'connect ETIMEDOUT' }); + sandbox.stub(axios, 'get').rejects(new Error('connect ETIMEDOUT')); + }); + + it('stays in the folder on the device configured under its address', async () => { + const devicesIds = []; + const allDevices = [{ name: '10.0.0.1:16127', deviceID: 'CONFIGURED-DEVICE' }]; + + const devices = await helpers.buildDeviceConfiguration( + [{ ip: '10.0.0.1:16127' }], LOCAL, 'MY-DEVICE', new Map(), [], devicesIds, allDevices, + ); + + expect(devices).to.deep.equal([{ deviceID: 'MY-DEVICE' }, { deviceID: 'CONFIGURED-DEVICE' }]); + expect(devicesIds, 'the sweep of unused devices would remove it').to.deep.equal(['CONFIGURED-DEVICE']); + }); + + it('leaves the folder while more than one device is configured under its address', async () => { + const allDevices = [ + { name: '10.0.0.1:16127', deviceID: 'CONFIGURED-DEVICE' }, + { name: '10.0.0.1:16127', deviceID: 'DEVICE-OF-A-NEIGHBOUR' }, + ]; + + const devices = await helpers.buildDeviceConfiguration( + [{ ip: '10.0.0.1:16127' }], LOCAL, 'MY-DEVICE', new Map(), [], [], allDevices, + ); + + expect(devices).to.deep.equal([{ deviceID: 'MY-DEVICE' }]); + }); + }); + it('configures a new peer device not to auto-accept folders', async () => { const devicesConfiguration = []; diff --git a/tests/unit/untilSucceeds.test.js b/tests/unit/untilSucceeds.test.js new file mode 100644 index 0000000000..20e0df8e3c --- /dev/null +++ b/tests/unit/untilSucceeds.test.js @@ -0,0 +1,38 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const proxyquire = require('proxyquire').noCallThru(); + +describe('untilSucceeds', () => { + let delay; + let untilSucceeds; + let BOOT_STEP_RETRY_MS; + + beforeEach(() => { + delay = sinon.stub().resolves(); + ({ untilSucceeds, BOOT_STEP_RETRY_MS } = proxyquire('../../ZelBack/src/services/utils/untilSucceeds', { + '../serviceHelper': { delay }, + '../../lib/log': { error: sinon.stub(), info: sinon.stub(), warn: sinon.stub() }, + })); + }); + + it('answers what the step resolved, without waiting, when it succeeds at once', async () => { + const step = sinon.stub().resolves('done'); + + expect(await untilSucceeds('a step', step)).to.equal('done'); + + sinon.assert.calledOnce(step); + sinon.assert.notCalled(delay); + }); + + it('runs a failing step again after each wait until it succeeds', async () => { + const step = sinon.stub(); + step.onCall(0).rejects(new Error('database did not answer')); + step.onCall(1).rejects(new Error('database did not answer')); + step.onCall(2).resolves('done'); + + expect(await untilSucceeds('a step', step)).to.equal('done'); + + sinon.assert.calledThrice(step); + expect(delay.args).to.deep.equal([[BOOT_STEP_RETRY_MS], [BOOT_STEP_RETRY_MS]]); + }); +}); diff --git a/tests/unit/updateAppGlobaly.test.js b/tests/unit/updateAppGlobaly.test.js index 5e4a80b9c7..baa187f639 100644 --- a/tests/unit/updateAppGlobaly.test.js +++ b/tests/unit/updateAppGlobaly.test.js @@ -107,4 +107,28 @@ describe('updateAppGlobaly tests', () => { ); expect(signatureCheck.called).to.equal(false); }); + + [ + ['an invalid app name', { name: 'Submit-App!' }], + ['an invalid owner', { owner: 'my-owner' }], + ].forEach(([description, change]) => { + it(`should refuse an enterprise update with ${description} before reading the spec`, async () => { + sinon.stub(daemonServiceMiscRpcs, 'isDaemonSynced').returns({ data: { synced: true, height: 3003100 } }); + + await advancedWorkflows.updateAppGlobaly({ + appSpecification: { + ...spec, version: 8, enterprise: 'blob', ...change, + }, + timestamp: Date.now(), + signature: 'sig', + type: 'fluxappupdate', + version: 1, + }).then( + () => expect.fail('should have refused'), + (error) => expect(error.message).to.equal('Invalid app name or owner'), + ); + sinon.assert.notCalled(appValidator.verifyAppSpecifications); + expect(signatureCheck.called).to.equal(false); + }); + }); }); diff --git a/tests/unit/upnpService.test.js b/tests/unit/upnpService.test.js index c73ca90245..b74d394869 100644 --- a/tests/unit/upnpService.test.js +++ b/tests/unit/upnpService.test.js @@ -44,7 +44,32 @@ const upnpService = proxyquire( { config, './fluxadmPort': fluxadmPortStub }, ); +// the options each nat-upnp client was constructed with +const clientOptions = []; +class RecordingClient extends natUpnp.Client { + constructor(options) { + super(options); + clientOptions.push(options); + } +} +proxyquire( + '../../ZelBack/src/services/upnpService', + { config, './fluxadmPort': fluxadmPortStub, '@runonflux/nat-upnp': { ...natUpnp, Client: RecordingClient } }, +); + +const GATEWAY_URL = 'http://192.168.1.1:5000/rootDesc.xml'; +const upnpServiceAtGatewayUrl = proxyquire( + '../../ZelBack/src/services/upnpService', + { config: { ...config, upnp: { gatewayUrl: GATEWAY_URL, nodeIp: '192.168.1.10' } }, './fluxadmPort': fluxadmPortStub }, +); + describe('upnpService tests', () => { + describe('client tests', () => { + it('should keep the router it discovers between calls', () => { + expect(clientOptions).to.deep.equal([{ cacheGateway: true }]); + }); + }); + describe('adjustFirewallForUPNP tests', () => { let originalUserConfig; @@ -100,6 +125,16 @@ describe('upnpService tests', () => { describe('verifyUPNPsupport tests', () => { let logSpy; + // what the router answers for the test mapping verifyUPNPsupport makes + const testMapping = { + public: { host: '', port: 16130 }, + private: { host: '192.168.1.10', port: 16130 }, + protocol: 'tcp', + enabled: true, + description: 'Flux_UPNP_Mapping_Test', + ttl: 0, + local: true, + }; beforeEach(() => { logSpy = sinon.spy(log, 'error'); @@ -113,7 +148,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -127,14 +162,29 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); }); - it('should hold the test port from the time it is mapped until it is removed, so a cleanup pass cannot take it mid-test', async () => { + it('should not wait for a turn when the router has room for the test mapping', async () => { + const delayStub = sinon.stub(serviceHelper, 'delay').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + const getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + + expect(await upnpService.verifyUPNPsupport(16177)).to.equal(true); + + expect(delayStub.getCalls().every((call) => call.args[0] < upnpService.BOOT_LISTING_STAGGER_MS)).to.equal(true); + sinon.assert.notCalled(getMappingsStub); + }); + + it('should hold the test port from the time it is mapped until it is read back, so a cleanup pass cannot take it mid-test', async () => { upnpService.mappedHere.clear(); const heldAt = {}; sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); sinon.stub(natUpnp.Client.prototype, 'createMapping').callsFake(async () => { heldAt.create = upnpService.mappedHere.has(16130); }); - sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves(true); - sinon.stub(natUpnp.Client.prototype, 'removeMapping').callsFake(async () => { heldAt.remove = upnpService.mappedHere.has(16130); }); + sinon.stub(natUpnp.Client.prototype, 'getMapping').callsFake(async () => { heldAt.readBack = upnpService.mappedHere.has(16130); return testMapping; }); + const removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); const clock = sinon.useFakeTimers(); @@ -143,7 +193,8 @@ describe('upnpService tests', () => { await clock.tickAsync(2_500); expect(await promise).to.equal(true); - expect(heldAt).to.deep.equal({ create: true, remove: true }); + expect(heldAt).to.deep.equal({ create: true, readBack: true }); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 16130, protocol: 'TCP' }); expect(upnpService.mappedHere.has(16130)).to.equal(false); }); @@ -151,7 +202,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').throws(); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -170,7 +221,7 @@ describe('upnpService tests', () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').throws(); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -188,8 +239,8 @@ describe('upnpService tests', () => { it('should log a proper error if createMapping throws', async () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'createMapping').throws(); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); + sinon.stub(natUpnp.Client.prototype, 'createMapping').rejects(new Error('connect ECONNREFUSED')); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -200,15 +251,257 @@ describe('upnpService tests', () => { const result = await promise; expect(result).to.equal(false); + sinon.assert.calledOnceWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping: TCP 16130 not mapped: connect ECONNREFUSED'); + }); + + describe('when the router\'s table is full', () => { + let createMappingStub; + let getMappingsStub; + let removeMappingStub; + + const listed = (port, description) => ({ + ...testMapping, public: { host: '', port }, private: { host: '192.168.1.10', port }, description, + }); + const tableFull = () => new natUpnp.UpnpError(728, 'NoPortMapsAvailable', 'AddPortMapping'); + + beforeEach(() => { + upnpService.mappedHere.clear(); + sinon.stub(serviceHelper, 'delay').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves('1.2.3.4'); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping'); + createMappingStub.onFirstCall().rejects(tableFull()); + createMappingStub.resolves(); + // the lookup after the refusal finds no entry; the one after the mapping finds it + sinon.stub(natUpnp.Client.prototype, 'getMapping').onFirstCall().resolves(null).resolves(testMapping); + getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([ + listed(31000, 'Flux_Test_App'), + listed(31001, 'Flux_Prelaunch_App_31001'), + listed(31002, 'Flux_App_web'), + listed(31003, 'Flux_Backend_API'), + // this check's own, held while it runs + listed(16130, 'Flux_UPNP_Mapping_Test'), + ]); + removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + }); + + it('should remove the test mappings earlier runs left, and nothing else, then map once more', async () => { + // the api port mapped here too: a test mapped on the retry is still checked and removed + natUpnp.Client.prototype.getMapping.withArgs({ public: 16127, protocol: 'TCP' }).resolves({ + ...testMapping, public: { host: '', port: 16127 }, private: { host: '192.168.1.10', port: 16127 }, + }); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(true); + sinon.assert.calledOnceWithExactly(getMappingsStub, { local: true, signal: sinon.match.instanceOf(AbortSignal) }); + const removedPorts = removeMappingStub.getCalls().map((call) => call.args[0].public.port ?? call.args[0].public); + expect(removedPorts).to.deep.equal([31000, 31001, 16130]); + sinon.assert.calledTwice(createMappingStub); + }); + + it('should count no refusal of its own test mapping toward the router\'s refusals', async () => { + const before = upnpService.routerRefusals(); + + expect(await upnpService.verifyUPNPsupport()).to.equal(true); + + sinon.assert.calledTwice(createMappingStub); + expect(upnpService.routerRefusals()).to.equal(before); + }); + + it('should count no refusal of its own test mapping when it fails, either', async () => { + getMappingsStub.resolves([]); + const before = upnpService.routerRefusals(); + + expect(await upnpService.verifyUPNPsupport()).to.equal(false); + + expect(upnpService.routerRefusals()).to.equal(before); + }); + + [ + [16127, 2], + [16137, 3], + [16177, 7], + [16187, 0], + [16197, 1], + ].forEach(([apiport, slot]) => { + it(`should wait slot ${slot}'s turn before listing the table on api port ${apiport}`, async () => { + getMappingsStub.resolves([]); + + await upnpService.verifyUPNPsupport(apiport); + + const wait = serviceHelper.delay.withArgs(slot * upnpService.BOOT_LISTING_STAGGER_MS); + sinon.assert.called(wait); + expect(wait.calledBefore(getMappingsStub), 'the wait comes before the listing').to.equal(true); + }); + }); + + it('should know the test mappings earlier runs left as a FRITZ!Box stores their descriptions', async () => { + getMappingsStub.resolves([ + listed(31000, 'Flux.Test.App'), + listed(31001, 'Flux.Prelaunch.App.31001'), + listed(31002, 'Flux.App.web'), + ]); + + expect(await upnpService.verifyUPNPsupport()).to.equal(true); + + const removedPorts = removeMappingStub.getCalls().map((call) => call.args[0].public.port ?? call.args[0].public); + expect(removedPorts.filter((port) => port !== 16130)).to.deep.equal([31000, 31001]); + }); + + it('should fail without mapping again when there is no test mapping to remove', async () => { + getMappingsStub.resolves([listed(31002, 'Flux_App_web')]); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.calledOnce(createMappingStub); + sinon.assert.calledWithMatch(logSpy, /^VerifyUPNPsupport - Failed Create Mapping: TCP 16130 not mapped: UPnP error 728/); + }); + + it('should fail without mapping again when the router cannot be listed', async () => { + getMappingsStub.rejects(new Error('listing timed out')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.calledOnce(createMappingStub); + }); + + it('should remove the left-overs, and map once more, when the router refuses with a generic 501 Action Failed', async () => { + createMappingStub.onFirstCall().rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'AddPortMapping')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(true); + sinon.assert.calledOnce(getMappingsStub); + sinon.assert.calledTwice(createMappingStub); + }); + + describe('and holds nothing of this node\'s to remove', () => { + // the router's entry for each port: this node's api port, or nothing + const routerMaps = (apiEntry) => { + natUpnp.Client.prototype.getMapping.reset(); + natUpnp.Client.prototype.getMapping.callsFake(async ({ public: port }) => (port === 16127 ? apiEntry : null)); + }; + const apiEntry = (overrides = {}) => ({ + ...testMapping, public: { host: '', port: 16127 }, private: { host: '192.168.1.10', port: 16127 }, description: 'Flux_Backend_API', ...overrides, + }); + + beforeEach(() => { + createMappingStub.reset(); + createMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'AddPortMapping')); + getMappingsStub.resolves([listed(31002, 'Flux_App_web')]); + }); + + it('should verify when the router already maps this node\'s api port to it', async () => { + routerMaps(apiEntry()); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(true); + expect(upnpService.isUPNP()).to.equal(true); + sinon.assert.calledWithExactly(natUpnp.Client.prototype.getMapping, { public: 16127, protocol: 'TCP' }); + sinon.assert.calledOnce(createMappingStub); + sinon.assert.notCalled(removeMappingStub); + expect(upnpService.mappedHere.has(16130)).to.equal(false); + }); + + [ + ['holds no mapping of the api port', null], + ['maps the api port to another machine', apiEntry({ local: false, private: { host: '192.168.1.11', port: 16127 } })], + ['maps the api port disabled', apiEntry({ enabled: false })], + ].forEach(([what, entry]) => { + it(`should fail when the router ${what}`, async () => { + routerMaps(entry); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + expect(upnpService.isUPNP()).to.equal(false); + }); + }); + + it('should fail on a refusal of the port\'s own, whatever the router maps', async () => { + routerMaps(apiEntry()); + createMappingStub.reset(); + createMappingStub.rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + }); + + it('should fail when the router does not answer, whatever it maps', async () => { + routerMaps(apiEntry()); + createMappingStub.reset(); + createMappingStub.rejects(new Error('timeout of 10000ms exceeded')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + }); + }); + + it('should list nothing when the router does not answer', async () => { + createMappingStub.onFirstCall().rejects(new Error('timeout of 10000ms exceeded')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.notCalled(getMappingsStub); + sinon.assert.calledOnce(createMappingStub); + }); + + it('should list nothing when the router refuses for the port\'s own reason', async () => { + createMappingStub.onFirstCall().rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + + const result = await upnpService.verifyUPNPsupport(); + + expect(result).to.equal(false); + sinon.assert.notCalled(getMappingsStub); + sinon.assert.calledOnce(createMappingStub); + }); + }); + + it('should look up the test mapping it made, by its port, and nothing else', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(true); + const getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); + const getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(true); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(2_500); + + expect(await promise).to.equal(true); + sinon.assert.calledOnceWithExactly(getMappingStub, { public: 16130, protocol: 'TCP' }); + sinon.assert.notCalled(getMappingsStub); + }); + + it('should fail when the router holds no mapping for the test port it just made', async () => { + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(true); + + const clock = sinon.useFakeTimers(); + const promise = upnpService.verifyUPNPsupport(16127); + await clock.tickAsync(2_500); + + expect(await promise).to.equal(false); sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Create Mapping'); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mapping'); }); - it('should log a proper error if getMappings throws', async () => { + it('should log a proper error if getMapping throws', async () => { sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').throws(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').throws(); sinon.stub(natUpnp.Client.prototype, 'removeMapping').returns(Promise.resolve(true)); const clock = sinon.useFakeTimers(); @@ -220,15 +513,17 @@ describe('upnpService tests', () => { expect(result).to.equal(false); sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mappings'); + sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed get Mapping'); }); - it('should log a proper error if removeMapping throws', async () => { - sinon.stub(natUpnp.Client.prototype, 'getPublicIp').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getGateway').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'createMapping').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'getMappings').returns(Promise.resolve(true)); - sinon.stub(natUpnp.Client.prototype, 'removeMapping').throws(); + it('should verify when the router refuses to remove the test mapping, and leave its removal pending', async () => { + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); + sinon.stub(natUpnp.Client.prototype, 'getPublicIp').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves(true); + sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(testMapping); + sinon.stub(natUpnp.Client.prototype, 'removeMapping').rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping')); const clock = sinon.useFakeTimers(); @@ -237,16 +532,18 @@ describe('upnpService tests', () => { await clock.tickAsync(2_500); const result = await promise; - expect(result).to.equal(false); - sinon.assert.calledTwice(logSpy); - sinon.assert.calledWithExactly(logSpy, 'VerifyUPNPsupport - Failed Remove Mapping'); + expect(result).to.equal(true); + expect([...upnpService.pendingRemovals.keys()]).to.deep.equal(['16130/TCP']); + expect(upnpService.removalPending()).to.equal(true); + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); }); }); describe('setupUPNP tests', () => { let logSpy; let createMappingSpy; - let getMappingsStub; + let getMappingStub; let removeMappingStub; let unitStateStub; @@ -259,7 +556,7 @@ describe('upnpService tests', () => { .withArgs('systemctl', sinon.match({ params: ['is-enabled', 'fluxadm-sshd.socket'] })); sshdSocket('disabled\n'); createMappingSpy = sinon.stub(natUpnp.Client.prototype, 'createMapping'); - getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings').resolves([]); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); @@ -301,18 +598,38 @@ describe('upnpService tests', () => { it('should remove its own maintenance ssh mapping while its socket is not enabled', async () => { createMappingSpy.returns(true); - getMappingsStub.resolves([fluxadmMapping()]); + getMappingStub.resolves(fluxadmMapping()); const result = await runSetup(123); expect(result).to.equal(true); sinon.assert.callCount(createMappingSpy, 4); + sinon.assert.calledOnceWithExactly(getMappingStub, { public: 118, protocol: 'TCP' }); sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); }); + it('should know its own maintenance ssh mapping as a FRITZ!Box stores its description', async () => { + createMappingSpy.returns(true); + getMappingStub.resolves(fluxadmMapping({ description: 'Flux.Fluxadm.SSH' })); + + await runSetup(123); + + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); + }); + + it('should remove nothing when the router holds no mapping of the maintenance ssh port', async () => { + createMappingSpy.returns(true); + + const result = await runSetup(123); + + expect(result).to.equal(true); + sinon.assert.calledOnce(getMappingStub); + sinon.assert.notCalled(removeMappingStub); + }); + it('should remove its own maintenance ssh mapping once its socket is gone, or systemctl cannot answer', async () => { createMappingSpy.returns(true); - getMappingsStub.resolves([fluxadmMapping()]); + getMappingStub.resolves(fluxadmMapping()); sshdSocket('', new Error('Failed to get unit file state for fluxadm-sshd.socket: No such file or directory')); const result = await runSetup(123); @@ -322,31 +639,34 @@ describe('upnpService tests', () => { sinon.assert.calledOnceWithExactly(removeMappingStub, { public: 118, protocol: 'TCP' }); }); - it('should keep a mapping of the same port that it did not make', async () => { - createMappingSpy.returns(true); - getMappingsStub.resolves([ - fluxadmMapping({ description: 'node owner ssh' }), - fluxadmMapping({ local: false }), - ]); + [ + ['made under another description', { description: 'node owner ssh' }], + ['to another machine', { local: false }], + ['the router cannot say is to this node', { local: null }], + ].forEach(([what, overrides]) => { + it(`should keep a mapping of the same port ${what}`, async () => { + createMappingSpy.returns(true); + getMappingStub.resolves(fluxadmMapping(overrides)); - const result = await runSetup(123); + const result = await runSetup(123); - expect(result).to.equal(true); - sinon.assert.calledOnce(getMappingsStub); - sinon.assert.notCalled(removeMappingStub); + expect(result).to.equal(true); + sinon.assert.calledOnce(getMappingStub); + sinon.assert.notCalled(removeMappingStub); + }); }); it('should neither map nor unmap the maintenance ssh port on ArcaneOS', async () => { createMappingSpy.returns(true); fluxadmPortStub.isArcane = true; sshdSocket('enabled\n'); - getMappingsStub.resolves([fluxadmMapping()]); + getMappingStub.resolves(fluxadmMapping()); const result = await runSetup(123); expect(result).to.equal(true); sinon.assert.callCount(createMappingSpy, 4); - sinon.assert.notCalled(getMappingsStub); + sinon.assert.notCalled(getMappingStub); sinon.assert.notCalled(removeMappingStub); }); @@ -376,16 +696,16 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); sinon.assert.callCount(createMappingSpy, 4); sinon.assert.calledWithExactly(createMappingSpy, { - public: 123, private: 123, ttl: 0, description: 'Flux_Backend_API', + public: 123, private: 123, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 124, private: 124, ttl: 0, description: 'Flux_Backend_API_SSL', + public: 124, private: 124, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API_SSL', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 122, private: 122, ttl: 0, description: 'Flux_Home_UI', + public: 122, private: 122, ttl: 0, protocol: 'TCP', description: 'Flux_Home_UI', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 125, private: 125, ttl: 0, description: 'Flux_Syncthing', + public: 125, private: 125, ttl: 0, protocol: 'TCP', description: 'Flux_Syncthing', }); }); @@ -403,19 +723,41 @@ describe('upnpService tests', () => { sinon.assert.notCalled(logSpy); sinon.assert.callCount(createMappingSpy, 4); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16127, private: 16127, ttl: 0, description: 'Flux_Backend_API', + public: 16127, private: 16127, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16128, private: 16128, ttl: 0, description: 'Flux_Backend_API_SSL', + public: 16128, private: 16128, ttl: 0, protocol: 'TCP', description: 'Flux_Backend_API_SSL', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16126, private: 16126, ttl: 0, description: 'Flux_Home_UI', + public: 16126, private: 16126, ttl: 0, protocol: 'TCP', description: 'Flux_Home_UI', }); sinon.assert.calledWithExactly(createMappingSpy, { - public: 16129, private: 16129, ttl: 0, description: 'Flux_Syncthing', + public: 16129, private: 16129, ttl: 0, protocol: 'TCP', description: 'Flux_Syncthing', }); }); + it('should take a core port the router refuses but already maps to this node as mapped', async () => { + createMappingSpy.withArgs(sinon.match({ public: 124 })).rejects(new natUpnp.UpnpError(728, 'NoPortMapsAvailable', 'AddPortMapping')); + getMappingStub.withArgs({ public: 124, protocol: 'TCP' }).resolves(fluxadmMapping({ + public: { host: '', port: 124 }, private: { host: '192.168.1.10', port: 124 }, description: 'Flux_Backend_API_SSL', enabled: true, + })); + + const result = await runSetup(123); + + expect(result).to.equal(true); + sinon.assert.notCalled(logSpy); + }); + + it('should fail on a core port the router refuses, naming it and the router\'s reason', async () => { + createMappingSpy.withArgs(sinon.match({ public: 124 })).rejects(new natUpnp.UpnpError(728, 'NoPortMapsAvailable', 'AddPortMapping')); + + const result = await runSetup(123); + + expect(result).to.equal(false); + sinon.assert.calledOnceWithExactly(logSpy, 'setupUPNP - TCP 124 not mapped: UPnP error 728: NoPortMapsAvailable (action: AddPortMapping)'); + sinon.assert.callCount(createMappingSpy, 2); + }); + it('should return error if client response throws', async () => { createMappingSpy.throws(); @@ -433,103 +775,689 @@ describe('upnpService tests', () => { describe('mapUpnpPort tests', () => { let logSpy; - let createMappingSpy; + let createMappingStub; + let getMappingStub; + + const refusal = (code, description) => new natUpnp.UpnpError(code, description, 'AddPortMapping'); + // the router's entry for port 123, to this node unless overridden + const entry = (overrides = {}) => ({ + public: { host: '', port: 123 }, + private: { host: '192.168.1.10', port: 123 }, + protocol: 'udp', + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + ...overrides, + }); beforeEach(() => { + upnpService.mappedHere.clear(); logSpy = sinon.spy(log, 'error'); - createMappingSpy = sinon.stub(natUpnp.Client.prototype, 'createMapping'); + sinon.stub(serviceHelper, 'delay').resolves(); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); }); afterEach(() => { sinon.restore(); }); - it('should return true if all client responses are valid', async () => { - createMappingSpy.returns(true); + it('should map TCP and then UDP, port for port, with no lease', async () => { + const result = await upnpService.mapUpnpPort(123, 'some description'); - const clock = sinon.useFakeTimers(); + expect(result).to.deep.equal({ + mapped: true, code: null, reason: null, routerWide: false, + }); + sinon.assert.notCalled(logSpy); + sinon.assert.calledTwice(createMappingStub); + sinon.assert.calledWithExactly(createMappingStub.firstCall, { + public: 123, private: 123, ttl: 0, protocol: 'TCP', description: 'some description', + }); + sinon.assert.calledWithExactly(createMappingStub.secondCall, { + public: 123, private: 123, ttl: 0, protocol: 'UDP', description: 'some description', + }); + }); - const promise = upnpService.mapUpnpPort(123, 'some description'); + it('should map only the protocols it is given', async () => { + upnpService.mappedProtocols.clear(); + const result = await upnpService.mapUpnpPort(123, 'Flux_Test_App', { protocols: ['TCP'] }); - await clock.tickAsync(1_000); - const result = await promise; + expect(result.mapped).to.equal(true); + sinon.assert.calledOnceWithExactly(createMappingStub, { + public: 123, private: 123, ttl: 0, protocol: 'TCP', description: 'Flux_Test_App', + }); + expect([...upnpService.mappedProtocols.get(123)]).to.deep.equal(['TCP']); + }); - expect(result).to.equal(true); - sinon.assert.notCalled(logSpy); - sinon.assert.calledTwice(createMappingSpy); - sinon.assert.calledWithExactly(createMappingSpy, { - public: 123, - private: 123, - ttl: 0, - protocol: 'TCP', - description: 'some description', + it('should signal each refusal it counts as the router\'s, and no other', async () => { + const onRefused = sinon.spy(); + upnpService.routerEvents.on('mappingRefused', onRefused); + try { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); + getMappingStub.resolves(entry({ private: { host: '192.168.1.20', port: 123 }, local: false })); + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + sinon.assert.notCalled(onRefused); + + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.resolves(null); + await upnpService.mapUpnpPort(124, 'Flux_App_web'); + await upnpService.mapUpnpPort(125, 'Flux_App_web'); + sinon.assert.calledTwice(onRefused); + } finally { + upnpService.routerEvents.off('mappingRefused', onRefused); + } + }); + + it('should not ask for UDP when TCP is refused', async () => { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result.mapped).to.equal(false); + sinon.assert.calledOnce(createMappingStub); + }); + + it('should give a port another machine holds as the port\'s refusal, not the router\'s', async () => { + createMappingStub.onSecondCall().rejects(refusal(718, 'ConflictInMappingEntry')); + getMappingStub.resolves(entry({ local: false, private: { host: '192.168.1.11', port: 123 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.deep.equal({ + mapped: false, + code: 718, + reason: 'UDP 123 not mapped: UPnP error 718: ConflictInMappingEntry (action: AddPortMapping)', + routerWide: false, }); - sinon.assert.calledWithExactly(createMappingSpy, { - public: 123, - private: 123, - ttl: 0, - protocol: 'UDP', - description: 'some description', + expect(upnpService.routerRefusals()).to.equal(before); + sinon.assert.calledOnceWithExactly(logSpy, `UPnP - ${result.reason}`); + }); + + it('should count a full table as the router\'s refusal', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 728, routerWide: true }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + }); + + it('should count a router that does not answer as the router\'s refusal, and not ask it again', async () => { + createMappingStub.rejects(new Error('timeout of 10000ms exceeded')); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.deep.equal({ + mapped: false, code: null, reason: 'TCP 123 not mapped: timeout of 10000ms exceeded', routerWide: true, }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + sinon.assert.notCalled(getMappingStub); }); - it('should return error if client response throws', async () => { - createMappingSpy.throws(); + [ + [501, 'ActionFailed'], + [606, 'ActionNotAuthorized'], + ].forEach(([code, description]) => { + it(`should count a ${code} ${description} as the router's refusal`, async () => { + createMappingStub.rejects(refusal(code, description)); + const before = upnpService.routerRefusals(); - const clock = sinon.useFakeTimers(); + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); - const promise = upnpService.mapUpnpPort(123); + expect(result).to.include({ mapped: false, code, routerWide: true }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + }); + }); - await clock.tickAsync(1_000); - const result = await promise; + [ + [402, 'InvalidArgs'], + [716, 'WildCardNotPermittedInExtPort'], + [718, 'ConflictInMappingEntry'], + [724, 'SamePortValuesRequired'], + [725, 'OnlyPermanentLeasesSupported'], + [726, 'RemoteHostOnlySupportsWildcard'], + [727, 'ExternalPortOnlySupportsWildcard'], + [729, 'ConflictWithOtherMechanisms'], + ].forEach(([code, description]) => { + it(`should give a ${code} ${description} as the port's refusal, not the router's`, async () => { + createMappingStub.rejects(refusal(code, description)); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); + }); - expect(result).to.equal(false); - sinon.assert.calledOnce(logSpy); + [ + [501, 'ActionFailed'], + [606, 'ActionNotAuthorized'], + ].forEach(([code, description]) => { + it(`should give a ${code} for a port another machine holds as the port's refusal, not the router's`, async () => { + createMappingStub.rejects(refusal(code, description)); + getMappingStub.resolves(entry({ local: false, private: { host: '192.168.1.11', port: 123 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); + }); + + it('should give a 501 for a port this node holds otherwise as the port\'s refusal, not the router\'s', async () => { + createMappingStub.rejects(refusal(501, 'ActionFailed')); + getMappingStub.resolves(entry({ private: { host: '192.168.1.10', port: 124 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 501, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + }); + + it('should count a 728 as the router\'s refusal even for a port another machine holds', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.resolves(entry({ local: false, private: { host: '192.168.1.11', port: 123 } })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 728, routerWide: true }); + expect(upnpService.routerRefusals()).to.equal(before + 1); + }); + + it('should take a refusal as mapped when the router already maps the port to this node', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.callsFake(async ({ protocol }) => entry({ protocol: protocol.toLowerCase() })); + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result.mapped).to.equal(true); + expect(upnpService.routerRefusals()).to.equal(before); + sinon.assert.calledWithExactly(getMappingStub.firstCall, { public: 123, protocol: 'TCP' }); + sinon.assert.calledWithExactly(getMappingStub.secondCall, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(logSpy); + }); + + [ + ['to another port of this node', { private: { host: '192.168.1.10', port: 124 } }], + ['disabled', { enabled: false }], + ['where this node cannot tell whose it is', { local: null }], + ].forEach(([what, overrides]) => { + it(`should keep the refusal when the router's entry is ${what}`, async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable')); + getMappingStub.resolves(entry(overrides)); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 728 }); + }); + }); + + it('should keep the refusal when the router cannot be asked for its entry', async () => { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry')); + getMappingStub.rejects(new Error('socket hang up')); + + const result = await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: 718 }); + }); + + it('should ask the router nothing for what is not a port, and count nothing', async () => { + const before = upnpService.routerRefusals(); + + const result = await upnpService.mapUpnpPort(Number.NaN, 'Flux_App_web'); + + expect(result).to.include({ mapped: false, code: null, routerWide: false }); + expect(upnpService.routerRefusals()).to.equal(before); + sinon.assert.notCalled(createMappingStub); + }); + + it('should no longer hold a port it failed to map', async () => { + createMappingStub.onSecondCall().rejects(refusal(718, 'ConflictInMappingEntry')); + + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(upnpService.mappedHere.has(123)).to.equal(false); + }); + + it('should hold a port it mapped', async () => { + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + expect(upnpService.mappedHere.has(123)).to.equal(true); }); }); describe('removeMapUpnpPort tests', () => { let logSpy; - let removeMappingSpy; + let createMappingStub; + let getMappingStub; + let removeMappingStub; + + const refusal = (code, description, action) => new natUpnp.UpnpError(code, description, action); + const removedProtocols = () => removeMappingStub.getCalls().map((call) => call.args[0].protocol); beforeEach(() => { + upnpService.mappedHere.clear(); + upnpService.mappedProtocols.clear(); logSpy = sinon.spy(log, 'error'); - removeMappingSpy = sinon.stub(natUpnp.Client.prototype, 'removeMapping'); + sinon.stub(serviceHelper, 'delay').resolves(); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(null); + removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); afterEach(() => { sinon.restore(); }); - it('should return true if all client responses are valid', async () => { - removeMappingSpy.returns(true); + it('should remove both protocols this node mapped, without asking the router whose they are', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + getMappingStub.resetHistory(); - const clock = sinon.useFakeTimers(); + const result = await upnpService.removeMapUpnpPort(123); - const promise = upnpService.removeMapUpnpPort(123, 'some description'); + expect(result).to.equal(true); + sinon.assert.notCalled(getMappingStub); + sinon.assert.calledWithExactly(removeMappingStub.firstCall, { public: 123, protocol: 'TCP' }); + sinon.assert.calledWithExactly(removeMappingStub.secondCall, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(logSpy); + }); - await clock.tickAsync(1_000); - const result = await promise; + it('should remove the TCP half of a port whose UDP the router refused', async () => { + createMappingStub.onSecondCall().rejects(refusal(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + + await upnpService.removeMapUpnpPort(123); + + expect(removedProtocols()).to.deep.equal(['TCP']); + }); + + it('should ask the router to remove nothing of a port it refused to map here', async () => { + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + + const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(true); - sinon.assert.notCalled(logSpy); - sinon.assert.calledTwice(removeMappingSpy); - sinon.assert.calledWithExactly(removeMappingSpy, { public: 123, protocol: 'TCP' }); - sinon.assert.calledWithExactly(removeMappingSpy, { public: 123, protocol: 'UDP' }); + sinon.assert.notCalled(removeMappingStub); }); - it('should return error if client response throws', async () => { - removeMappingSpy.throws(); + it('should ask the router to remove nothing of a port this node never mapped', async () => { + const result = await upnpService.removeMapUpnpPort(123); - const clock = sinon.useFakeTimers(); + expect(result).to.equal(true); + sinon.assert.notCalled(removeMappingStub); + }); - const promise = upnpService.removeMapUpnpPort(123); + it('should forget a protocol whose latest map the router refused', async () => { + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + createMappingStub.rejects(refusal(718, 'ConflictInMappingEntry', 'AddPortMapping')); + await upnpService.mapUpnpPort(123, 'Flux_App_web'); - await clock.tickAsync(1_000); - const result = await promise; + await upnpService.removeMapUpnpPort(123); + + // TCP was refused; UDP, not asked again, stays as the first map left it + expect(removedProtocols()).to.deep.equal(['UDP']); + }); + + it('should count a refusal the router has the port mapped here for as mapped here', async () => { + createMappingStub.rejects(refusal(728, 'NoPortMapsAvailable', 'AddPortMapping')); + getMappingStub.callsFake(async ({ protocol }) => ({ + public: { host: '', port: 123 }, + private: { host: '192.168.1.10', port: 123 }, + protocol: protocol.toLowerCase(), + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + })); + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + await upnpService.removeMapUpnpPort(123); + + expect(removedProtocols()).to.deep.equal(['TCP', 'UDP']); + }); + + it('should remove UDP when the router refuses to remove TCP, and report the port not removed', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + removeMappingStub.withArgs({ public: 123, protocol: 'TCP' }).rejects(refusal(501, 'ActionFailed', 'DeletePortMapping')); + + const result = await upnpService.removeMapUpnpPort(123); expect(result).to.equal(false); - sinon.assert.calledOnce(logSpy); + expect(removedProtocols()).to.deep.equal(['TCP', 'UDP']); + sinon.assert.calledOnceWithExactly(logSpy, 'UPnP - TCP 123 not unmapped: UPnP error 501: ActionFailed (action: DeletePortMapping)'); + }); + + it('should take a removal the router has no entry for as removed', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + removeMappingStub.rejects(refusal(714, 'NoSuchEntryInArray', 'DeletePortMapping')); + + const result = await upnpService.removeMapUpnpPort(123); + + expect(result).to.equal(true); + sinon.assert.notCalled(logSpy); + }); + + it('should ask a second removal of the same port for nothing', async () => { + await upnpService.mapUpnpPort(123, 'Flux_Test_App'); + await upnpService.removeMapUpnpPort(123); + removeMappingStub.resetHistory(); + + await upnpService.removeMapUpnpPort(123); + + sinon.assert.notCalled(removeMappingStub); + }); + }); + + describe('pending removal tests', () => { + const FIRST_MS = 30 * 1000; + const MAX_MS = 60 * 1000; + let clock; + let createMappingStub; + let getMappingStub; + let removeMappingStub; + + const refused = () => new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping'); + // the router's entry for a port, to this node unless overridden + const entry = (port, protocol, overrides = {}) => ({ + public: { host: '', port }, + private: { host: '192.168.1.10', port }, + protocol: protocol.toLowerCase(), + enabled: true, + description: 'Flux_App_web', + ttl: 0, + local: true, + ...overrides, + }); + const lookups = () => getMappingStub.getCalls().map((call) => `${call.args[0].public}/${call.args[0].protocol}`); + const removals = () => removeMappingStub.getCalls().map((call) => `${call.args[0].public}/${call.args[0].protocol}`); + // a test port mapped TCP only: its UDP refused, held by another machine + const mapTcpOnly = async (port, description) => { + createMappingStub.withArgs(sinon.match({ public: port, protocol: 'UDP' })).rejects(new natUpnp.UpnpError(718, 'ConflictInMappingEntry', 'AddPortMapping')); + getMappingStub.withArgs(sinon.match({ public: port, protocol: 'UDP' })) + .resolves(entry(port, 'UDP', { private: { host: '192.168.1.11', port }, local: false })); + await upnpService.mapUpnpPort(port, description); + getMappingStub.resetHistory(); + }; + + beforeEach(() => { + upnpService.mappedHere.clear(); + upnpService.mappedProtocols.clear(); + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); + sinon.stub(serviceHelper, 'delay').resolves(); + sinon.stub(log, 'error'); + sinon.stub(log, 'info'); + createMappingStub = sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + getMappingStub = sinon.stub(natUpnp.Client.prototype, 'getMapping') + .callsFake(async ({ public: port, protocol }) => entry(port, protocol)); + removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); + clock = sinon.useFakeTimers(); + }); + + afterEach(() => { + upnpService.pendingRemovals.forEach((pending) => clearTimeout(pending.timer)); + upnpService.pendingRemovals.clear(); + clock.restore(); + sinon.restore(); + }); + + it('should ask the router again for a removal it refused, waiting longer each time, until it goes', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + removeMappingStub.onCall(2).rejects(refused()); + + expect(await upnpService.removeMapUpnpPort(123, 'Flux_Test_App')).to.equal(false); + expect(removals()).to.deep.equal(['123/TCP']); + + await clock.tickAsync(FIRST_MS - 1); + expect(removals().length, 'nothing before the first wait').to.equal(1); + await clock.tickAsync(1); + expect(removals().length, 'asked again after the first wait').to.equal(2); + await clock.tickAsync(MAX_MS - 1); + expect(removals().length, 'the wait doubles').to.equal(2); + await clock.tickAsync(1); + expect(removals().length).to.equal(3); + await clock.tickAsync(MAX_MS); + expect(removals().length, 'removed on the fourth ask, the wait capped').to.equal(4); + expect(upnpService.pendingRemovals.size).to.equal(0); + + await clock.tickAsync(MAX_MS * 2); + expect(removals().length, 'nothing once removed').to.equal(4); + }); + + it('should never give up, and wait no longer than the cap between asks', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + // 30s, then 1m each + await clock.tickAsync(FIRST_MS); + expect(removals().length).to.equal(2); + await clock.tickAsync(MAX_MS - 1); + expect(removals().length).to.equal(2); + await clock.tickAsync(1); + expect(removals().length).to.equal(3); + await clock.tickAsync(MAX_MS * 10); + expect(removals().length, 'still asking').to.equal(13); + expect(upnpService.pendingRemovals.has('123/TCP')).to.equal(true); + }); + + it('should take a router that holds no such entry as removed', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(new natUpnp.UpnpError(714, 'NoSuchEntryInArray', 'DeletePortMapping')); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + await clock.tickAsync(FIRST_MS); + expect(upnpService.pendingRemovals.size).to.equal(0); + await clock.tickAsync(MAX_MS); + expect(removals().length).to.equal(2); + }); + + it('should drop a pending removal of a port this node maps again', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + await clock.tickAsync(MAX_MS); + + expect(removals()).to.deep.equal(['123/TCP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should ask for pending removals one at a time', async () => { + await mapTcpOnly(123, 'Flux_Test_App'); + await mapTcpOnly(124, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_Test_App'); + await upnpService.removeMapUpnpPort(124, 'Flux_Test_App'); + let inFlight = 0; + let most = 0; + removeMappingStub.callsFake(async () => { + inFlight += 1; + most = Math.max(most, inFlight); + await new Promise((resolve) => { setTimeout(resolve, 1000); }); + inFlight -= 1; + }); + + await clock.tickAsync(FIRST_MS + 5000); + + expect(removals().slice(2).sort()).to.deep.equal(['123/TCP', '124/TCP']); + expect(most, 'never two at once').to.equal(1); + }); + + it('should keep asking for both protocols of a port when the router refuses to remove either', async () => { + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + expect([...upnpService.pendingRemovals.keys()].sort()).to.deep.equal(['123/TCP', '123/UDP']); + await clock.tickAsync(FIRST_MS); + expect(removals().slice(2).sort()).to.deep.equal(['123/TCP', '123/UDP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should keep a pending removal when the port is removed again with nothing mapped since', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + expect(await upnpService.removeMapUpnpPort(123, 'Flux_App_web')).to.equal(true); + await clock.tickAsync(FIRST_MS); + + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should say a removal is pending until the router has removed it, whatever the mapping', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + expect(upnpService.removalPending()).to.equal(true); + + await clock.tickAsync(FIRST_MS); + expect(upnpService.removalPending(), 'removed on the next ask').to.equal(false); + }); + + it('should say pending when the first removal is refused, and clear when the last goes', async () => { + const events = []; + const onPending = () => events.push('pending'); + const onClear = () => events.push('clear'); + upnpService.routerEvents.on('removalPending', onPending); + upnpService.routerEvents.on('removalsClear', onClear); + try { + await mapTcpOnly(123, 'Flux_App_web'); + await mapTcpOnly(124, 'Flux_Test_App'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(refused()); + removeMappingStub.onCall(2).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + await upnpService.removeMapUpnpPort(124, 'Flux_Test_App'); + expect(events, 'once, for the first').to.deep.equal(['pending']); + + await clock.tickAsync(FIRST_MS); + expect(events, 'one still pending').to.deep.equal(['pending']); + await clock.tickAsync(MAX_MS); + expect(events).to.deep.equal(['pending', 'clear']); + } finally { + upnpService.routerEvents.off('removalPending', onPending); + upnpService.routerEvents.off('removalsClear', onClear); + } + }); + + it('should clear when the port of the last pending removal is mapped again', async () => { + const onClear = sinon.spy(); + upnpService.routerEvents.on('removalsClear', onClear); + try { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + + await upnpService.mapUpnpPort(123, 'Flux_App_web'); + + sinon.assert.calledOnce(onClear); + expect(upnpService.removalPending()).to.equal(false); + } finally { + upnpService.routerEvents.off('removalsClear', onClear); + } + }); + + it('should look the entry up before each ask, and remove it only while it is this node\'s', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + expect(lookups(), 'the first removal asks nothing first').to.deep.equal([]); + + await clock.tickAsync(FIRST_MS); + + expect(lookups()).to.deep.equal(['123/TCP']); + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.pendingRemovals.size).to.equal(0); + }); + + it('should end a removal with nothing sent when the router no longer holds the entry, whatever it answers to a removal', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.resolves(null); + + await clock.tickAsync(FIRST_MS); + + expect(removals(), 'no removal after the lookup').to.deep.equal(['123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + await clock.tickAsync(MAX_MS * 5); + expect(lookups(), 'asked nothing more').to.have.length(1); + }); + + it('should end a removal with nothing sent when the router holds the port for something else', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.callsFake(async ({ public: port, protocol }) => entry(port, protocol, { private: { host: '192.168.1.11', port }, local: false })); + + await clock.tickAsync(FIRST_MS); + + expect(removals()).to.deep.equal(['123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + }); + + it('should send the removal when the router answers the lookup with a fault, and keep it pending while the removal is refused', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'GetSpecificPortMappingEntry')); + + await clock.tickAsync(FIRST_MS); + + expect(removals(), 'a lookup it did not settle leaves the mapping counted as this node\'s').to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.removalPending()).to.equal(true); + + removeMappingStub.resolves(); + await clock.tickAsync(MAX_MS); + expect(removals()).to.have.length(3); + expect(upnpService.removalPending(), 'ended once the router removed it').to.equal(false); + }); + + it('should end a removal whose lookup the router answers with a fault once it answers the removal with no such entry', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.onCall(0).rejects(refused()); + removeMappingStub.onCall(1).rejects(new natUpnp.UpnpError(714, 'NoSuchEntryInArray', 'DeletePortMapping')); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.rejects(new natUpnp.UpnpError(501, 'ActionFailed', 'GetSpecificPortMappingEntry')); + + await clock.tickAsync(FIRST_MS); + + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.removalPending()).to.equal(false); + }); + + it('should send the removal while the router does not answer the lookup, and keep it pending while the removal is refused', async () => { + await mapTcpOnly(123, 'Flux_App_web'); + removeMappingStub.rejects(refused()); + await upnpService.removeMapUpnpPort(123, 'Flux_App_web'); + getMappingStub.rejects(new Error('timeout of 10000ms exceeded')); + + await clock.tickAsync(FIRST_MS); + expect(removals()).to.deep.equal(['123/TCP', '123/TCP']); + expect(upnpService.removalPending()).to.equal(true); + + removeMappingStub.resolves(); + await clock.tickAsync(MAX_MS); + expect(upnpService.removalPending(), 'ended once the router removed it').to.equal(false); }); }); @@ -554,7 +1482,8 @@ describe('upnpService tests', () => { describe('removeStaleMappings tests', () => { const BUDGET_MS = 75 * 1000; - let runStub; + const LISTING_MS = 60 * 1000; + let getMappingsStub; let removeMappingStub; let clock; let table; @@ -570,33 +1499,27 @@ describe('upnpService tests', () => { ...overrides, }); - // the router's error reply to an index past the last entry - const arrayIndexInvalid = () => Object.assign(new Error('Request failed with status code 500'), { response: { status: 500 } }); - const asEntry = (m) => ({ - GetGenericPortMappingEntryResponse: { - NewRemoteHost: m.public.host, - NewExternalPort: String(m.public.port), - NewProtocol: m.protocol.toUpperCase(), - NewInternalPort: String(m.private.port), - NewInternalClient: m.private.host, - NewEnabled: '1', - NewPortMappingDescription: m.description, - NewLeaseDuration: '0', - }, + const refused = () => new natUpnp.UpnpError(501, 'ActionFailed', 'DeletePortMapping'); + // a request the router never answers: it ends when its signal aborts, as + // the library's requests do + const unanswered = ({ signal }) => new Promise((resolve, reject) => { + signal.addEventListener('abort', () => reject(signal.reason), { once: true }); }); const routerHas = (mappings) => { table = mappings; }; beforeEach(() => { upnpService.mappedHere.clear(); clock = sinon.useFakeTimers(); + // the fake clock does not drive AbortSignal.timeout's own timer + sinon.stub(AbortSignal, 'timeout').callsFake((ms) => { + const controller = new AbortController(); + setTimeout(() => controller.abort(new DOMException('The operation was aborted due to timeout', 'TimeoutError')), ms); + return controller.signal; + }); sinon.stub(serviceHelper, 'delay').resolves(); table = []; - runStub = sinon.stub().callsFake(async (action, [[, index]]) => { - if (index >= table.length) throw arrayIndexInvalid(); - return asEntry(table[index]); - }); - sinon.stub(natUpnp.Client.prototype, 'getGateway').resolves({ gateway: { run: runStub }, address: '192.168.1.10' }); - sinon.stub(natUpnp.Client.prototype, 'getMappings').rejects(new Error('the pass walks the table itself')); + getMappingsStub = sinon.stub(natUpnp.Client.prototype, 'getMappings') + .callsFake(async (options) => table.filter((m) => !options.local || m.local === true)); removeMappingStub = sinon.stub(natUpnp.Client.prototype, 'removeMapping').resolves(); }); @@ -605,25 +1528,49 @@ describe('upnpService tests', () => { sinon.restore(); }); - it('should walk the router\'s table one index at a time, up to the error reply that ends it', async () => { - routerHas([mapping(31000, 'Flux_App_gone'), mapping(31001, 'Flux_App_sibling', { private: { host: '192.168.1.11', port: 31001 } })]); + it('should list only the router\'s mappings to this node, and remove the stale one', async () => { + routerHas([mapping(31000, 'Flux_App_gone'), mapping(31001, 'Flux_App_sibling', { local: false, private: { host: '192.168.1.11', port: 31001 } })]); const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); - expect(runStub.getCalls().map((call) => [call.args[0], call.args[1][0][1]])).to.deep.equal([ - ['GetGenericPortMappingEntry', 0], - ['GetGenericPortMappingEntry', 1], - ['GetGenericPortMappingEntry', 2], + sinon.assert.calledOnceWithExactly(getMappingsStub, { local: true, signal: sinon.match.instanceOf(AbortSignal) }); + expect(removed).to.equal(1); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP', signal: sinon.match.instanceOf(AbortSignal) }); + }); + + it('should know its own mappings as a FRITZ!Box stores their descriptions, and nothing else', async () => { + // FRITZ!Box rewrites the punctuation: 'Flux_App_gone' comes back 'Flux.App.gone' + routerHas([ + mapping(31000, 'Flux.App.gone'), + mapping(31001, 'Flux.Prelaunch.App.31001'), + mapping(31002, 'Flux.Test.App'), + mapping(31003, 'Flux.manual.entry'), + mapping(31004, 'FluxOS.Reserved'), ]); + + const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); + + expect(removed).to.equal(3); + const removedPorts = removeMappingStub.getCalls().map((call) => call.args[0].public.port); + expect(removedPorts).to.deep.equal([31000, 31001, 31002]); + }); + + it('should give the router\'s requests whole-millisecond timeouts from a budget that halves to a fraction', async () => { + routerHas([mapping(31000, 'Flux_App_gone')]); + + const removed = await upnpService.removeStaleMappings(async () => [16127], 29_999); + expect(removed).to.equal(1); - sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'TCP' }); + const timeouts = AbortSignal.timeout.getCalls().map((call) => call.args[0]); + expect(timeouts).to.have.length(2); + timeouts.forEach((ms) => expect(Number.isInteger(ms), `${ms} ms`).to.equal(true)); }); - it('should take an error reply at the first index for an empty table', async () => { + it('should remove nothing from an empty table', async () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); expect(removed).to.equal(0); - sinon.assert.calledOnce(runStub); + sinon.assert.calledOnce(getMappingsStub); sinon.assert.notCalled(removeMappingStub); }); @@ -633,7 +1580,7 @@ describe('upnpService tests', () => { const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); expect(removed).to.equal(1); - sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'UDP' }); + sinon.assert.calledOnceWithExactly(removeMappingStub, { public: { host: '', port: 31000 }, protocol: 'UDP', signal: sinon.match.instanceOf(AbortSignal) }); }); it('should keep every mapping on a port something holds, whatever its description', async () => { @@ -660,9 +1607,8 @@ describe('upnpService tests', () => { expect(removeMappingStub.getCalls().map((call) => call.args[0].public.port)).to.deep.equal([16127, 16128, 16126, 16129]); }); - it('should never touch a mapping to another address, the operator\'s manual entries or anything it did not make', async () => { + it('should never touch the operator\'s manual entries or anything it did not make', async () => { routerHas([ - mapping(31000, 'Flux_App_sibling', { local: false, private: { host: '192.168.1.11', port: 31000 } }), mapping(31001, 'Flux_manual_entry'), mapping(22, 'ssh'), mapping(31002, undefined), @@ -687,7 +1633,7 @@ describe('upnpService tests', () => { it('should go on past a removal the router refuses', async () => { routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_Prelaunch_App_31001')]); - removeMappingStub.onFirstCall().rejects(new Error('ActionFailed')); + removeMappingStub.onFirstCall().rejects(refused()); const warnSpy = sinon.spy(log, 'warn'); const removed = await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS); @@ -697,37 +1643,24 @@ describe('upnpService tests', () => { sinon.assert.calledOnce(warnSpy); }); - it('should reject, and remove nothing, when the router does not reply to the listing at all', async () => { - routerHas([mapping(31000, 'Flux_App_gone'), mapping(31001, 'Flux_App_gone2')]); - runStub.onSecondCall().rejects(Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' })); + it('should reject, and remove nothing, when the listing fails', async () => { + getMappingsStub.rejects(Object.assign(new Error('socket hang up'), { code: 'ECONNRESET' })); let error; await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS).catch((err) => { error = err; }); expect(error.message).to.equal('socket hang up'); - sinon.assert.calledTwice(runStub); - sinon.assert.notCalled(removeMappingStub); - }); - - it('should reject on a reply that is not a mapping entry', async () => { - runStub.resolves({ SomethingElse: {} }); - - let error; - await upnpService.removeStaleMappings(async () => [16127], BUDGET_MS).catch((err) => { error = err; }); - - expect(error.message).to.equal('listing the mappings: incorrect response'); sinon.assert.notCalled(removeMappingStub); }); it('should ask what is held only after the router has been listed', async () => { routerHas([mapping(31000, 'Flux_App_a')]); - let requestsBeforeHeld; - const held = sinon.stub().callsFake(async () => { requestsBeforeHeld = runStub.callCount; return [16127]; }); + let listingsBeforeHeld; + const held = sinon.stub().callsFake(async () => { listingsBeforeHeld = getMappingsStub.callCount; return [16127]; }); await upnpService.removeStaleMappings(held, BUDGET_MS); - expect(requestsBeforeHeld).to.equal(2); - expect(runStub.callCount).to.equal(2); + expect(listingsBeforeHeld).to.equal(1); }); it('should remove nothing when what is held cannot be read', async () => { @@ -774,8 +1707,9 @@ describe('upnpService tests', () => { it('should remove a port mapUpnpPort mapped once removeMapUpnpPort has given it up, even if the router refused', async () => { sinon.stub(natUpnp.Client.prototype, 'createMapping').resolves(); + sinon.stub(natUpnp.Client.prototype, 'getMapping').resolves(mapping(31000, 'Flux_App_gone')); await upnpService.mapUpnpPort(31000, 'Flux_App_gone'); - removeMappingStub.onFirstCall().rejects(new Error('ActionFailed')); + removeMappingStub.onFirstCall().rejects(refused()); await upnpService.removeMapUpnpPort(31000); removeMappingStub.reset(); removeMappingStub.resolves(); @@ -787,53 +1721,42 @@ describe('upnpService tests', () => { sinon.assert.calledOnce(removeMappingStub); }); - it('should send the router nothing more once a listing request goes unanswered', async () => { - routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b')]); - runStub.onSecondCall().returns(new Promise(() => {})); + it('should give the listing its 60s of the 75s slot, and remove nothing when it does not finish', async () => { + routerHas([mapping(31000, 'Flux_App_a')]); + getMappingsStub.callsFake(unanswered); let error; const passing = upnpService.removeStaleMappings(async () => [], BUDGET_MS).catch((err) => { error = err; }); - await clock.tickAsync(BUDGET_MS); + await clock.tickAsync(LISTING_MS - 1); + expect(error).to.equal(undefined); + await clock.tickAsync(1); await passing; - expect(error.message).to.match(/listing the mappings: the router did not answer within 15s/); - sinon.assert.calledTwice(runStub); + expect(error.name).to.equal('TimeoutError'); sinon.assert.notCalled(removeMappingStub); }); - it('should stop walking a slow router\'s table before the slot ends, and remove from what it listed', async () => { - // 100 entries at 1s each would take 100s; the listing has 60s of the 75s slot - routerHas(Array.from({ length: 100 }, (_, i) => mapping(31000 + i, `Flux_App_${i}`))); - runStub.callsFake(async (action, args) => { - await new Promise((resolve) => { setTimeout(resolve, 1000); }); - const [[, index]] = args; - if (index >= table.length) throw arrayIndexInvalid(); - return asEntry(table[index]); - }); + it('should step past a refused removal but end the pass on one the router does not answer', async () => { + routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b'), mapping(31002, 'Flux_App_c')]); + removeMappingStub.onFirstCall().rejects(refused()); + removeMappingStub.onSecondCall().callsFake(unanswered); const passing = upnpService.removeStaleMappings(async () => [], BUDGET_MS); await clock.tickAsync(BUDGET_MS); const removed = await passing; - // no request goes out after the listing's 60s: at most the one in flight at that moment - expect(runStub.callCount).to.be.within(60, 61); - expect(removed).to.equal(runStub.callCount); - const requestsBySlotEnd = runStub.callCount; - await clock.tickAsync(10 * 60 * 1000); - expect(runStub.callCount).to.equal(requestsBySlotEnd); + expect(removed).to.equal(0); + sinon.assert.calledTwice(removeMappingStub); }); - it('should step past a refused removal but end the pass on one the router does not answer', async () => { - routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b'), mapping(31002, 'Flux_App_c')]); - removeMappingStub.onFirstCall().rejects(new Error('ActionFailed')); - removeMappingStub.onSecondCall().returns(new Promise(() => {})); + it('should end the pass on a removal that fails without a UPnP error', async () => { + routerHas([mapping(31000, 'Flux_App_a'), mapping(31001, 'Flux_App_b')]); + removeMappingStub.onFirstCall().rejects(Object.assign(new Error('connect ECONNREFUSED'), { code: 'ECONNREFUSED' })); - const passing = upnpService.removeStaleMappings(async () => [], BUDGET_MS); - await clock.tickAsync(15 * 1000); - const removed = await passing; + const removed = await upnpService.removeStaleMappings(async () => [], BUDGET_MS); expect(removed).to.equal(0); - sinon.assert.calledTwice(removeMappingStub); + sinon.assert.calledOnce(removeMappingStub); }); it('should stop removing once its budget is spent, and report what it did', async () => { @@ -1318,23 +2241,32 @@ describe('upnpService tests', () => { }); }); - it('should show a proper message if all data is valid', async () => { - getGatewaySpy.resolves('10.1.1.1'); + it('should answer with the gateway\'s description and services and this node\'s address', async () => { + const device = new natUpnp.Device(GATEWAY_URL); + getGatewaySpy.resolves(new natUpnp.UpnpInfo(device, '192.168.1.10')); verifyPrivilegeStub.resolves(true); - const req = { - params: { - port: '1234', - }, - query: { - test: 'test', - }, - }; const res = generateResponse(); - await upnpService.getGatewayApi(req, res); + await upnpService.getGatewayApi({ params: {}, query: {} }, res); sinon.assert.calledOnce(getGatewaySpy); - sinon.assert.calledOnceWithExactly(res.json, { status: 'success', data: '10.1.1.1' }); + sinon.assert.calledOnceWithExactly(res.json, { + status: 'success', + data: { gateway: { description: GATEWAY_URL, services: device.services }, address: '192.168.1.10' }, + }); + }); + + it('should reach the router at config.upnp.gatewayUrl as config.upnp.nodeIp, without discovering it', async () => { + getGatewaySpy.restore(); + verifyPrivilegeStub.resolves(true); + const res = generateResponse(); + + await upnpServiceAtGatewayUrl.getGatewayApi({ params: {}, query: {} }, res); + + sinon.assert.calledOnceWithExactly(res.json, { + status: 'success', + data: { gateway: { description: GATEWAY_URL, services: new natUpnp.Device(GATEWAY_URL).services }, address: '192.168.1.10' }, + }); }); }); }); diff --git a/tests/unit/withinDeadline.test.js b/tests/unit/withinDeadline.test.js new file mode 100644 index 0000000000..457a7ce411 --- /dev/null +++ b/tests/unit/withinDeadline.test.js @@ -0,0 +1,38 @@ +const { expect } = require('chai'); +const sinon = require('sinon'); +const { withinDeadline, TIMED_OUT } = require('../../ZelBack/src/services/utils/withinDeadline'); + +describe('withinDeadline tests', () => { + let clock; + + beforeEach(() => { + clock = sinon.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); + }); + + afterEach(() => { + clock.restore(); + }); + + it('answers what the work resolved, when it resolves in time', async () => { + expect(await withinDeadline(Promise.resolve('done'), 1000)).to.equal('done'); + }); + + it('rejects with the work\'s own error, when it rejects in time', async () => { + const failure = await withinDeadline(Promise.reject(new Error('refused')), 1000).catch((error) => error); + + expect(failure.message).to.equal('refused'); + }); + + it('answers TIMED_OUT once the deadline passes first', async () => { + const answer = withinDeadline(new Promise(() => {}), 1000); + await clock.tickAsync(1000); + + expect(await answer).to.equal(TIMED_OUT); + }); + + it('leaves no timer behind once the work resolves', async () => { + await withinDeadline(Promise.resolve('done'), 1000); + + expect(clock.countTimers()).to.equal(0); + }); +});