From 903f930df969583b26d973e7a9675ac6021b62b4 Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Fri, 2 Oct 2026 12:40:38 -0400 Subject: [PATCH] feature: clean up removed game server nodes --- .../game-server-node.module.ts | 12 + .../game-server-node.service.ts | 6 + .../jobs/CleanupRemovedNodes.spec.ts | 250 ++++++++++++++++++ .../jobs/CleanupRemovedNodes.ts | 214 +++++++++++++++ 4 files changed, 482 insertions(+) create mode 100644 src/game-server-node/jobs/CleanupRemovedNodes.spec.ts create mode 100644 src/game-server-node/jobs/CleanupRemovedNodes.ts diff --git a/src/game-server-node/game-server-node.module.ts b/src/game-server-node/game-server-node.module.ts index 004340a3..636dd7dc 100644 --- a/src/game-server-node/game-server-node.module.ts +++ b/src/game-server-node/game-server-node.module.ts @@ -33,6 +33,7 @@ import { PluginRuntimeModule } from "src/plugin-runtime/plugin-runtime.module"; import { GamePluginsModule } from "src/game-plugins/game-plugins.module"; import { BakeShaders } from "./jobs/BakeShaders"; import { ValidateGamedata } from "./jobs/ValidateGamedata"; +import { CleanupRemovedNodes } from "./jobs/CleanupRemovedNodes"; import { MapAssetsModule } from "src/map-assets/map-assets.module"; import { PostgresModule } from "src/postgres/postgres.module"; @@ -47,6 +48,7 @@ import { PostgresModule } from "src/postgres/postgres.module"; CheckServerPluginVersions, BakeShaders, ValidateGamedata, + CleanupRemovedNodes, ...getQueuesProcessors("GameServerNode"), loggerFactory(), ], @@ -145,6 +147,16 @@ export class GameServerNodeModule implements OnApplicationBootstrap { }, }, ); + + void queue.add( + CleanupRemovedNodes.name, + {}, + { + repeat: { + pattern: "*/5 * * * *", + }, + }, + ); } public async onApplicationBootstrap() { diff --git a/src/game-server-node/game-server-node.service.ts b/src/game-server-node/game-server-node.service.ts index 9b0a03e7..a3d4df95 100644 --- a/src/game-server-node/game-server-node.service.ts +++ b/src/game-server-node/game-server-node.service.ts @@ -313,6 +313,12 @@ export class GameServerNodeService { this.logger.log(`Creating volumes for node ${node}`); await this.createVolumes(node); } + // A node that registers again after its removal was cleaned up still has + // its CS:GO install on disk, but no volume for it anymore. + if (csgoBuildId && !game_server_nodes_by_pk) { + this.logger.log(`Creating CS:GO volume for node ${node}`); + await this.createVolumes(node, "csgo"); + } if (game_server_nodes_by_pk?.status === "NotAcceptingNewMatches") { status = "NotAcceptingNewMatches"; } diff --git a/src/game-server-node/jobs/CleanupRemovedNodes.spec.ts b/src/game-server-node/jobs/CleanupRemovedNodes.spec.ts new file mode 100644 index 00000000..c197204b --- /dev/null +++ b/src/game-server-node/jobs/CleanupRemovedNodes.spec.ts @@ -0,0 +1,250 @@ +const mockCore = { + listNode: jest.fn(), + deleteNode: jest.fn(), + listPersistentVolume: jest.fn(), + deletePersistentVolume: jest.fn(), + readNamespacedPersistentVolumeClaim: jest.fn(), + deleteNamespacedPersistentVolumeClaim: jest.fn(), +}; +const mockBatch = { + listNamespacedJob: jest.fn(), + deleteNamespacedJob: jest.fn(), +}; + +jest.mock("@kubernetes/client-node", () => ({ + BatchV1Api: class BatchV1Api {}, + CoreV1Api: class CoreV1Api {}, + KubeConfig: class KubeConfig { + loadFromDefault() {} + makeApiClient(api: { name: string }) { + return api.name === "BatchV1Api" ? mockBatch : mockCore; + } + }, +})); + +import { CleanupRemovedNodes } from "./CleanupRemovedNodes"; + +const minutesAgo = (minutes: number) => + new Date(Date.now() - minutes * 60 * 1000).toISOString(); + +const node = ( + name: string, + { + ready = "False", + since = minutesAgo(30), + labels = {}, + }: { ready?: string; since?: string; labels?: Record } = {}, +) => ({ + metadata: { + name, + uid: `uid-${name}`, + labels: { "5stack-id": name, ...labels }, + creationTimestamp: minutesAgo(60), + }, + status: { + conditions: [{ type: "Ready", status: ready, lastTransitionTime: since }], + }, +}); + +const job = (name: string, app: string, host: string) => ({ + metadata: { name, uid: `uid-${name}` }, + spec: { + template: { + metadata: { labels: { app } }, + spec: { + affinity: { + nodeAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { + key: "kubernetes.io/hostname", + operator: "In", + values: [host], + }, + ], + }, + ], + }, + }, + }, + }, + }, + }, +}); + +const volume = (name: string, id: string, spec: Record = {}) => ({ + metadata: { name, uid: `uid-${name}` }, + spec: { + storageClassName: "local-storage", + persistentVolumeReclaimPolicy: "Retain", + claimRef: { namespace: "5stack", name: `${name}-claim` }, + nodeAffinity: { + required: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { key: "5stack-id", operator: "In", values: [id] }, + ], + }, + ], + }, + }, + ...spec, + }, +}); + +describe("CleanupRemovedNodes", () => { + let logger: { log: jest.Mock; warn: jest.Mock }; + let cleanup: CleanupRemovedNodes; + + const setup = ({ + rows = [] as string[], + nodes = [] as unknown[], + jobs = [] as unknown[], + volumes = [] as unknown[], + }) => { + mockCore.listNode.mockResolvedValue({ items: nodes }); + mockBatch.listNamespacedJob.mockResolvedValue({ items: jobs }); + mockCore.listPersistentVolume.mockResolvedValue({ items: volumes }); + mockCore.readNamespacedPersistentVolumeClaim.mockImplementation( + async ({ name }: { name: string }) => ({ + metadata: { uid: `uid-${name}` }, + spec: { volumeName: name.replace(/-claim$/, "") }, + }), + ); + + logger = { log: jest.fn(), warn: jest.fn() }; + cleanup = new CleanupRemovedNodes( + { + query: jest.fn().mockResolvedValue({ + game_server_nodes: rows.map((id) => ({ id })), + }), + } as any, + { get: () => ({ namespace: "5stack" }) } as any, + logger as any, + ); + }; + + const deleted = (mock: jest.Mock) => + mock.mock.calls.map(([{ name }]) => name); + + beforeEach(() => { + jest.clearAllMocks(); + for (const mock of [ + mockCore.deleteNode, + mockCore.deletePersistentVolume, + mockCore.deleteNamespacedPersistentVolumeClaim, + mockBatch.deleteNamespacedJob, + ]) { + mock.mockResolvedValue({}); + } + }); + + it("deletes only the Nodes of removed nodes that stayed NotReady", async () => { + setup({ + rows: ["has-row"], + nodes: [ + node("removed"), + node("has-row"), + node("still-ready", { ready: "True" }), + node("just-went-down", { since: minutesAgo(2) }), + node("panel", { + labels: { "node-role.kubernetes.io/control-plane": "true" }, + }), + ], + }); + + await cleanup.process(); + + expect(deleted(mockCore.deleteNode)).toEqual(["removed"]); + expect(mockCore.deleteNode).toHaveBeenCalledWith({ + name: "removed", + body: { preconditions: { uid: "uid-removed" } }, + }); + }); + + it("deletes the jobs, claims and volumes of a node with no row and no Node", async () => { + setup({ + rows: ["has-row"], + nodes: [node("still-registered", { ready: "True" })], + jobs: [ + job("update-cs-server-gone", "update-cs-server", "gone"), + job("streamer-gone", "game-streamer", "gone"), + job("match-gone", "game-server", "gone"), + job("update-cs-server-has-row", "update-cs-server", "has-row"), + job( + "update-cs-server-registered", + "update-cs-server", + "still-registered", + ), + ], + volumes: [ + volume("serverfiles-gone", "gone"), + volume("demos-gone", "gone", { + persistentVolumeReclaimPolicy: "Delete", + }), + volume("steamcmd-gone", "gone", { + claimRef: { namespace: "other", name: "steamcmd-gone-claim" }, + }), + volume("serverfiles-has-row", "has-row"), + volume("serverfiles-still-registered", "still-registered"), + ], + }); + + await cleanup.process(); + + expect(deleted(mockBatch.deleteNamespacedJob)).toEqual([ + "update-cs-server-gone", + "streamer-gone", + ]); + expect(mockBatch.deleteNamespacedJob).toHaveBeenCalledWith({ + name: "update-cs-server-gone", + namespace: "5stack", + body: { + propagationPolicy: "Background", + preconditions: { uid: "uid-update-cs-server-gone" }, + }, + }); + expect(deleted(mockCore.deleteNamespacedPersistentVolumeClaim)).toEqual([ + "serverfiles-gone-claim", + ]); + expect(deleted(mockCore.deletePersistentVolume)).toEqual([ + "serverfiles-gone", + ]); + }); + + it("keeps a volume's claim when it is bound to another volume", async () => { + setup({ + volumes: [volume("serverfiles-gone", "gone", { claimRef: undefined })], + }); + mockCore.readNamespacedPersistentVolumeClaim.mockResolvedValue({ + metadata: { uid: "uid-claim" }, + spec: { volumeName: "something-else" }, + }); + + await cleanup.process(); + + expect( + mockCore.deleteNamespacedPersistentVolumeClaim, + ).not.toHaveBeenCalled(); + expect(deleted(mockCore.deletePersistentVolume)).toEqual([ + "serverfiles-gone", + ]); + }); + + it("ignores objects that are already gone and explains a forbidden Node delete", async () => { + setup({ + nodes: [node("removed")], + volumes: [volume("serverfiles-gone", "gone")], + }); + mockCore.deleteNode.mockRejectedValue({ code: 403 }); + mockCore.deletePersistentVolume.mockRejectedValue({ code: 404 }); + + await cleanup.process(); + + expect(logger.warn).toHaveBeenCalledTimes(1); + expect(logger.warn.mock.calls[0][0]).toContain("run ./update.sh"); + }); +}); diff --git a/src/game-server-node/jobs/CleanupRemovedNodes.ts b/src/game-server-node/jobs/CleanupRemovedNodes.ts new file mode 100644 index 00000000..86cb1d92 --- /dev/null +++ b/src/game-server-node/jobs/CleanupRemovedNodes.ts @@ -0,0 +1,214 @@ +import { WorkerHost } from "@nestjs/bullmq"; +import { Logger } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { + BatchV1Api, + CoreV1Api, + KubeConfig, + V1Node, + V1NodeSelectorTerm, +} from "@kubernetes/client-node"; +import { HasuraService } from "src/hasura/hasura.service"; +import { GameServersConfig } from "src/configs/types/GameServersConfig"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { GameServerQueues } from "../enums/GameServerQueues"; + +// A host that went down may only be restarting, and once its Node is deleted +// its kubelet does not register again until k3s-agent restarts. +const NOT_READY_GRACE_MS = 10 * 60 * 1000; + +// Jobs the api pins to a node for the node itself. Match server jobs are left +// alone. +const NODE_JOB_APPS = [ + "update-cs-server", + "validate-gamedata", + "game-streamer", +]; + +/** + * Cleans up after game server nodes that were removed from the panel. A node + * counts as removed once it has no game_server_nodes row; a host that is still + * running re-creates its row on its next ping. + * + * 1. The Node of a removed node is deleted once it has been NotReady for + * NOT_READY_GRACE_MS. Control plane Nodes are never deleted. + * 2. Jobs, claims and volumes pinned to a node id that has neither a row nor a + * Node are deleted. Files on the node's disk are kept (the volumes are + * Retain). A Node deleted in step 1 is still counted here, so its pods are + * gone by the next run. + */ +@UseQueue("GameServerNode", GameServerQueues.GameUpdate) +export class CleanupRemovedNodes extends WorkerHost { + private readonly namespace: string; + + constructor( + private readonly hasura: HasuraService, + private readonly config: ConfigService, + private readonly logger: Logger, + ) { + super(); + this.namespace = + this.config.get("gameServers").namespace; + } + + async process(): Promise { + const kc = new KubeConfig(); + kc.loadFromDefault(); + const core = kc.makeApiClient(CoreV1Api); + const batch = kc.makeApiClient(BatchV1Api); + + const { game_server_nodes } = await this.hasura.query({ + game_server_nodes: { id: true }, + }); + const rows = new Set(game_server_nodes.map(({ id }) => id)); + + const { items: nodes } = await core.listNode(); + const present = new Set( + nodes.flatMap((node) => [ + node.metadata.name, + node.metadata.labels?.["5stack-id"], + ]), + ); + const isRemoved = (id?: string) => + !!id && !rows.has(id) && !present.has(id); + + for (const node of nodes) { + if (this.isDeadRemovedNode(node, rows)) { + await this.remove(`Node ${node.metadata.name}`, () => + core.deleteNode({ + name: node.metadata.name, + body: { preconditions: { uid: node.metadata.uid } }, + }), + ); + } + } + + const { items: jobs } = await batch.listNamespacedJob({ + namespace: this.namespace, + }); + for (const job of jobs) { + const template = job.spec?.template; + if ( + job.metadata.deletionTimestamp || + !NODE_JOB_APPS.includes(template?.metadata?.labels?.app) || + !isRemoved( + pinnedTo( + template?.spec?.affinity?.nodeAffinity + ?.requiredDuringSchedulingIgnoredDuringExecution + ?.nodeSelectorTerms, + "kubernetes.io/hostname", + ), + ) + ) { + continue; + } + await this.remove(`Job ${job.metadata.name}`, () => + batch.deleteNamespacedJob({ + name: job.metadata.name, + namespace: this.namespace, + body: { + propagationPolicy: "Background", + preconditions: { uid: job.metadata.uid }, + }, + }), + ); + } + + const { items: volumes } = await core.listPersistentVolume(); + for (const volume of volumes) { + const name = volume.metadata.name; + const claimName = `${name}-claim`; + const claimRef = volume.spec?.claimRef; + if ( + volume.metadata.deletionTimestamp || + volume.spec?.storageClassName !== "local-storage" || + volume.spec.persistentVolumeReclaimPolicy !== "Retain" || + (claimRef && + (claimRef.namespace !== this.namespace || + claimRef.name !== claimName)) || + !isRemoved( + pinnedTo( + volume.spec.nodeAffinity?.required?.nodeSelectorTerms, + "5stack-id", + ), + ) + ) { + continue; + } + + const claim = await core + .readNamespacedPersistentVolumeClaim({ + name: claimName, + namespace: this.namespace, + }) + .catch((): undefined => undefined); + if (claim?.spec?.volumeName === name) { + await this.remove(`PersistentVolumeClaim ${claimName}`, () => + core.deleteNamespacedPersistentVolumeClaim({ + name: claimName, + namespace: this.namespace, + body: { preconditions: { uid: claim.metadata.uid } }, + }), + ); + } + + await this.remove(`PersistentVolume ${name}`, () => + core.deletePersistentVolume({ + name, + body: { preconditions: { uid: volume.metadata.uid } }, + }), + ); + } + } + + private isDeadRemovedNode(node: V1Node, rows: Set) { + const labels = node.metadata.labels ?? {}; + const id = labels["5stack-id"]; + if ( + !id || + rows.has(id) || + "node-role.kubernetes.io/control-plane" in labels || + "node-role.kubernetes.io/master" in labels + ) { + return false; + } + + const ready = node.status?.conditions?.find(({ type }) => type === "Ready"); + if (ready?.status === "True") { + return false; + } + + const since = new Date( + ready?.lastTransitionTime ?? node.metadata.creationTimestamp, + ).getTime(); + return Date.now() - since >= NOT_READY_GRACE_MS; + } + + private async remove(what: string, remove: () => Promise) { + try { + await remove(); + this.logger.log(`[node cleanup] removed ${what}`); + } catch (error) { + const code = Number(error?.code); + // already gone, or replaced by a newer object of the same name + if (code === 404 || code === 409) { + return; + } + this.logger.warn( + `[node cleanup] could not remove ${what}: ${ + code === 403 + ? "forbidden, run ./update.sh on the panel to update the api's permissions" + : error?.message + }`, + ); + } + } +} + +function pinnedTo(terms: V1NodeSelectorTerm[] | undefined, key: string) { + return terms + ?.flatMap((term) => term.matchExpressions ?? []) + .find( + (expression) => expression.key === key && expression.operator === "In", + )?.values?.[0]; +}