From 03b8748b8039bcde295677d874893ff49a3e811d Mon Sep 17 00:00:00 2001 From: Luke Policinski Date: Mon, 28 Sep 2026 08:35:40 -0400 Subject: [PATCH] feature: map assets builds --- hasura/metadata/actions.graphql | 6 + hasura/metadata/actions.yaml | 8 + .../tables/public_map_asset_builds.yaml | 21 + .../tables/public_utility_drift_scans.yaml | 1 + .../databases/default/tables/tables.yaml | 1 + .../1887000000000_map_asset_builds/down.sql | 1 + .../1887000000000_map_asset_builds/up.sql | 16 + .../down.sql | 2 + .../up.sql | 2 + hasura/triggers/map_asset_builds.sql | 5 + .../game-server-node.module.ts | 2 + .../game-server-node.service.ts | 26 +- .../jobs/ValidateGamedata.spec.ts | 59 ++ src/game-server-node/jobs/ValidateGamedata.ts | 22 + src/map-assets/enums/MapAssetsQueues.ts | 3 + src/map-assets/jobs/BuildMapAssets.ts | 39 + src/map-assets/map-assets.controller.ts | 15 + src/map-assets/map-assets.module.ts | 35 + src/map-assets/map-assets.service.spec.ts | 552 +++++++++++++ src/map-assets/map-assets.service.ts | 722 ++++++++++++++++++ src/system/enums/SystemSettingName.ts | 4 + src/utilities/QueueProcessors.ts | 3 +- src/utility/utility-callouts.service.spec.ts | 167 ++++ src/utility/utility-callouts.service.ts | 101 ++- src/utility/utility-drift.service.ts | 14 +- test/map-asset-builds.spec.ts | 281 +++++++ test/utility-drift.spec.ts | 32 +- 27 files changed, 2119 insertions(+), 21 deletions(-) create mode 100644 hasura/metadata/databases/default/tables/public_map_asset_builds.yaml create mode 100644 hasura/migrations/default/1887000000000_map_asset_builds/down.sql create mode 100644 hasura/migrations/default/1887000000000_map_asset_builds/up.sql create mode 100644 hasura/migrations/default/1887000000100_utility_drift_scan_caveats/down.sql create mode 100644 hasura/migrations/default/1887000000100_utility_drift_scan_caveats/up.sql create mode 100644 hasura/triggers/map_asset_builds.sql create mode 100644 src/game-server-node/jobs/ValidateGamedata.spec.ts create mode 100644 src/map-assets/enums/MapAssetsQueues.ts create mode 100644 src/map-assets/jobs/BuildMapAssets.ts create mode 100644 src/map-assets/map-assets.controller.ts create mode 100644 src/map-assets/map-assets.module.ts create mode 100644 src/map-assets/map-assets.service.spec.ts create mode 100644 src/map-assets/map-assets.service.ts create mode 100644 test/map-asset-builds.spec.ts diff --git a/hasura/metadata/actions.graphql b/hasura/metadata/actions.graphql index c489d7cf3..c9835db06 100644 --- a/hasura/metadata/actions.graphql +++ b/hasura/metadata/actions.graphql @@ -84,6 +84,12 @@ type Mutation { ): SuccessOutput } +type Mutation { + buildMapAssets( + game_server_node_id: uuid! + ): SuccessOutput +} + type Mutation { callForOrganizer( match_id: String! diff --git a/hasura/metadata/actions.yaml b/hasura/metadata/actions.yaml index a82594ea4..90a9e0d7b 100644 --- a/hasura/metadata/actions.yaml +++ b/hasura/metadata/actions.yaml @@ -165,6 +165,14 @@ actions: permissions: - role: administrator comment: Launch a Vulkan shader pre-bake Job on a GPU node + - name: buildMapAssets + definition: + kind: synchronous + handler: '{{HASURA_GRAPHQL_ACTIONS_HOOK}}' + forward_client_headers: true + permissions: + - role: administrator + comment: Build and publish map assets from a node's CS2 install (5stack.gg only) - name: callForOrganizer definition: kind: synchronous diff --git a/hasura/metadata/databases/default/tables/public_map_asset_builds.yaml b/hasura/metadata/databases/default/tables/public_map_asset_builds.yaml new file mode 100644 index 000000000..7ff8762cc --- /dev/null +++ b/hasura/metadata/databases/default/tables/public_map_asset_builds.yaml @@ -0,0 +1,21 @@ +table: + name: map_asset_builds + schema: public +select_permissions: + - role: administrator + permission: + columns: + - build_id + - status + - started_at + - finished_at + - manifest + - maps + - failed + - failed_view + - error + - created_at + - updated_at + filter: {} + allow_aggregations: true + comment: "" diff --git a/hasura/metadata/databases/default/tables/public_utility_drift_scans.yaml b/hasura/metadata/databases/default/tables/public_utility_drift_scans.yaml index 75f6452a3..3254f81ef 100644 --- a/hasura/metadata/databases/default/tables/public_utility_drift_scans.yaml +++ b/hasura/metadata/databases/default/tables/public_utility_drift_scans.yaml @@ -31,6 +31,7 @@ select_permissions: - broken - unsimulatable - max_distance + - caveats - requested_by_steam_id - started_at - finished_at diff --git a/hasura/metadata/databases/default/tables/tables.yaml b/hasura/metadata/databases/default/tables/tables.yaml index cff6b5f3a..157416d96 100644 --- a/hasura/metadata/databases/default/tables/tables.yaml +++ b/hasura/metadata/databases/default/tables/tables.yaml @@ -102,6 +102,7 @@ - "!include public_league_teams.yaml" - "!include public_lobbies.yaml" - "!include public_lobby_players.yaml" +- "!include public_map_asset_builds.yaml" - "!include public_map_callouts.yaml" - "!include public_map_pools.yaml" - "!include public_maps.yaml" diff --git a/hasura/migrations/default/1887000000000_map_asset_builds/down.sql b/hasura/migrations/default/1887000000000_map_asset_builds/down.sql new file mode 100644 index 000000000..996b8e69a --- /dev/null +++ b/hasura/migrations/default/1887000000000_map_asset_builds/down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS "public"."map_asset_builds"; diff --git a/hasura/migrations/default/1887000000000_map_asset_builds/up.sql b/hasura/migrations/default/1887000000000_map_asset_builds/up.sql new file mode 100644 index 000000000..5ed3c9bbe --- /dev/null +++ b/hasura/migrations/default/1887000000000_map_asset_builds/up.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS "public"."map_asset_builds" ( + "build_id" text NOT NULL, + "status" text NOT NULL DEFAULT 'Pending', + "started_at" timestamptz, + "finished_at" timestamptz, + "manifest" text, + "maps" jsonb, + "failed" jsonb, + "failed_view" jsonb, + "error" text, + "created_at" timestamptz NOT NULL DEFAULT now(), + "updated_at" timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY ("build_id"), + CONSTRAINT "map_asset_builds_status_check" + CHECK ("status" IN ('Pending', 'Building', 'Published', 'Partial', 'Failed')) +); diff --git a/hasura/migrations/default/1887000000100_utility_drift_scan_caveats/down.sql b/hasura/migrations/default/1887000000100_utility_drift_scan_caveats/down.sql new file mode 100644 index 000000000..dc729d6b3 --- /dev/null +++ b/hasura/migrations/default/1887000000100_utility_drift_scan_caveats/down.sql @@ -0,0 +1,2 @@ +ALTER TABLE "public"."utility_drift_scans" + DROP COLUMN IF EXISTS "caveats"; diff --git a/hasura/migrations/default/1887000000100_utility_drift_scan_caveats/up.sql b/hasura/migrations/default/1887000000100_utility_drift_scan_caveats/up.sql new file mode 100644 index 000000000..68623877f --- /dev/null +++ b/hasura/migrations/default/1887000000100_utility_drift_scan_caveats/up.sql @@ -0,0 +1,2 @@ +ALTER TABLE "public"."utility_drift_scans" + ADD COLUMN IF NOT EXISTS "caveats" jsonb; diff --git a/hasura/triggers/map_asset_builds.sql b/hasura/triggers/map_asset_builds.sql new file mode 100644 index 000000000..23aaf5290 --- /dev/null +++ b/hasura/triggers/map_asset_builds.sql @@ -0,0 +1,5 @@ +DROP TRIGGER IF EXISTS "set_public_map_asset_builds_updated_at" ON "public"."map_asset_builds"; +CREATE TRIGGER "set_public_map_asset_builds_updated_at" +BEFORE UPDATE ON "public"."map_asset_builds" +FOR EACH ROW +EXECUTE PROCEDURE "public"."set_current_timestamp_updated_at"(); diff --git a/src/game-server-node/game-server-node.module.ts b/src/game-server-node/game-server-node.module.ts index 89b67b917..0b12ae57f 100644 --- a/src/game-server-node/game-server-node.module.ts +++ b/src/game-server-node/game-server-node.module.ts @@ -33,6 +33,7 @@ import { PluginRuntimeModule } from "src/plugin-runtime/plugin-runtime.module"; import { GamePluginsModule } from "src/game-plugins/game-plugins.module"; import { BakeShaders } from "./jobs/BakeShaders"; import { ValidateGamedata } from "./jobs/ValidateGamedata"; +import { MapAssetsModule } from "src/map-assets/map-assets.module"; @Module({ providers: [ @@ -59,6 +60,7 @@ import { ValidateGamedata } from "./jobs/ValidateGamedata"; GameStreamerModule, PluginRuntimeModule, GamePluginsModule, + MapAssetsModule, BullModule.registerQueue( { name: GameServerQueues.GameUpdate, diff --git a/src/game-server-node/game-server-node.service.ts b/src/game-server-node/game-server-node.service.ts index f1595ab31..89e404760 100644 --- a/src/game-server-node/game-server-node.service.ts +++ b/src/game-server-node/game-server-node.service.ts @@ -24,6 +24,7 @@ import { GameServerQueues } from "./enums/GameServerQueues"; import { NotificationsService } from "src/notifications/notifications.service"; import { PluginRuntimeService } from "src/plugin-runtime/plugin-runtime.service"; import { PluginRuntime } from "src/configs/types/GameServersConfig"; +import { MapAssetsService } from "src/map-assets/map-assets.service"; export type GamedataValidationRuntime = PluginRuntime; @@ -73,6 +74,7 @@ export class GameServerNodeService { protected readonly loggingService: LoggingService, protected readonly notifications: NotificationsService, protected readonly pluginRuntimeService: PluginRuntimeService, + protected readonly mapAssets: MapAssetsService, @InjectQueue(GameServerQueues.ValidateGamedata) private readonly validateGamedataQueue: Queue, ) { @@ -347,7 +349,18 @@ export class GameServerNodeService { csBulid && game_server_nodes_by_pk.build_id !== csBulid ) { - await this.queueGamedataValidation(node, csBulid); + // Map assets and the gamedata validator both read this node's install, + // so when a validation is queued the map-assets build is chained after it + // (see ValidateGamedata) rather than run beside it on the same node. + const validating = await this.queueGamedataValidation(node, csBulid); + if (!validating) { + await this.mapAssets.queueBuild(node, csBulid).catch((error) => { + this.logger.warn( + `[map-assets] unable to queue build ${csBulid}`, + error, + ); + }); + } } if (transitionedFromOffline && game_server_nodes_by_pk.build_id) { @@ -942,14 +955,14 @@ export class GameServerNodeService { private async queueGamedataValidation( gameServerNodeId: string, buildId: number, - ) { + ): Promise { if (process.env.WEB_DOMAIN !== "5stack.gg") { - return; + return false; } const currentBuild = await this.getCurrentBuild(); if (buildId !== currentBuild) { - return; + return false; } const { gamedata_signature_validations } = await this.hasura.query({ @@ -966,7 +979,7 @@ export class GameServerNodeService { }); if (gamedata_signature_validations.length > 0) { - return; + return false; } await this.validateGamedataQueue.add( @@ -974,6 +987,7 @@ export class GameServerNodeService { { gameServerNodeId, buildId, + buildMapAssets: true, }, { jobId: `validate.${buildId}.auto`, @@ -982,6 +996,8 @@ export class GameServerNodeService { removeOnFail: true, }, ); + + return true; } public async validateGamedata( diff --git a/src/game-server-node/jobs/ValidateGamedata.spec.ts b/src/game-server-node/jobs/ValidateGamedata.spec.ts new file mode 100644 index 000000000..057242ddf --- /dev/null +++ b/src/game-server-node/jobs/ValidateGamedata.spec.ts @@ -0,0 +1,59 @@ +import { ValidateGamedata } from "./ValidateGamedata"; + +describe("ValidateGamedata", () => { + let gameServerNodeService: { validateGamedata: jest.Mock }; + let mapAssets: { queueBuild: jest.Mock }; + let job: ValidateGamedata; + + beforeEach(() => { + gameServerNodeService = { + validateGamedata: jest + .fn() + .mockResolvedValue({ status: "pass", broken: [] }), + }; + mapAssets = { queueBuild: jest.fn().mockResolvedValue(true) }; + job = new ValidateGamedata( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + { send: jest.fn() } as any, + gameServerNodeService as any, + mapAssets as any, + ); + }); + + const run = (data: Record) => + job.process({ + data: { gameServerNodeId: "node-1", buildId: 25537370, ...data }, + } as any); + + it("starts the chained map-assets build once validation finishes", async () => { + const order: Array = []; + gameServerNodeService.validateGamedata.mockImplementation(async () => { + order.push("validate"); + return { status: "pass", broken: [] }; + }); + mapAssets.queueBuild.mockImplementation(async () => { + order.push("map-assets"); + return true; + }); + + await run({ buildMapAssets: true }); + + expect(order).toEqual(["validate", "map-assets"]); + expect(mapAssets.queueBuild).toHaveBeenCalledWith("node-1", 25537370); + }); + + it("still starts it when validation fails outright", async () => { + gameServerNodeService.validateGamedata.mockRejectedValue( + new Error("k8s down"), + ); + + await expect(run({ buildMapAssets: true })).rejects.toThrow("k8s down"); + expect(mapAssets.queueBuild).toHaveBeenCalledWith("node-1", 25537370); + }); + + it("leaves map assets alone for a validation nobody chained to", async () => { + await run({}); + + expect(mapAssets.queueBuild).not.toHaveBeenCalled(); + }); +}); diff --git a/src/game-server-node/jobs/ValidateGamedata.ts b/src/game-server-node/jobs/ValidateGamedata.ts index f9a88c338..c2d3b2787 100644 --- a/src/game-server-node/jobs/ValidateGamedata.ts +++ b/src/game-server-node/jobs/ValidateGamedata.ts @@ -9,11 +9,13 @@ import { GameServerNodeService, GamedataValidationEntry, } from "../game-server-node.service"; +import { MapAssetsService } from "src/map-assets/map-assets.service"; type ValidateGamedataData = { gameServerNodeId: string; buildId: number; branch?: string; + buildMapAssets?: boolean; }; const GAMEDATA_ROUTING = { @@ -36,11 +38,31 @@ export class ValidateGamedata extends WorkerHost { protected readonly logger: Logger, protected readonly notifications: NotificationsService, protected readonly gameServerNodeService: GameServerNodeService, + protected readonly mapAssets: MapAssetsService, ) { super(); } + // The map-assets build for a new CS2 version waits for this validation, + // pass or fail, so the two never share the node's install at once. async process(job: Job): Promise { + try { + await this.validate(job); + } finally { + if (job.data.buildMapAssets) { + await this.mapAssets + .queueBuild(job.data.gameServerNodeId, job.data.buildId) + .catch((error) => { + this.logger.warn( + `[map-assets] unable to queue build ${job.data.buildId}`, + error, + ); + }); + } + } + } + + private async validate(job: Job): Promise { const { gameServerNodeId, buildId } = job.data; const branch = job.data.branch ?? "public"; diff --git a/src/map-assets/enums/MapAssetsQueues.ts b/src/map-assets/enums/MapAssetsQueues.ts new file mode 100644 index 000000000..16bfaaf08 --- /dev/null +++ b/src/map-assets/enums/MapAssetsQueues.ts @@ -0,0 +1,3 @@ +export enum MapAssetsQueues { + BuildMapAssets = "build-map-assets", +} diff --git a/src/map-assets/jobs/BuildMapAssets.ts b/src/map-assets/jobs/BuildMapAssets.ts new file mode 100644 index 000000000..09895ea07 --- /dev/null +++ b/src/map-assets/jobs/BuildMapAssets.ts @@ -0,0 +1,39 @@ +import { WorkerHost } from "@nestjs/bullmq"; +import { Job } from "bullmq"; +import { Logger } from "@nestjs/common"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { MapAssetsQueues } from "../enums/MapAssetsQueues"; +import { MapAssetsService } from "../map-assets.service"; + +type BuildMapAssetsData = { + gameServerNodeId: string; + buildId: string; +}; + +@UseQueue("MapAssets", MapAssetsQueues.BuildMapAssets) +export class BuildMapAssets extends WorkerHost { + constructor( + protected readonly logger: Logger, + protected readonly mapAssets: MapAssetsService, + ) { + super(); + } + + async process(job: Job): Promise { + const { gameServerNodeId } = job.data; + const buildId = String(job.data.buildId); + + const outcome = await this.mapAssets.build(gameServerNodeId, buildId); + + if (outcome.status === "Failed") { + this.logger.error( + `[map-assets] build ${buildId} failed on ${gameServerNodeId}: ${outcome.error}`, + ); + return; + } + + this.logger.log( + `[map-assets] published build ${buildId} (${Object.keys(outcome.maps ?? {}).length} maps)`, + ); + } +} diff --git a/src/map-assets/map-assets.controller.ts b/src/map-assets/map-assets.controller.ts new file mode 100644 index 000000000..2abb791e5 --- /dev/null +++ b/src/map-assets/map-assets.controller.ts @@ -0,0 +1,15 @@ +import { Controller } from "@nestjs/common"; +import { HasuraAction } from "../hasura/hasura.controller"; +import { MapAssetsService } from "./map-assets.service"; + +@Controller("map-assets") +export class MapAssetsController { + constructor(private readonly mapAssets: MapAssetsService) {} + + @HasuraAction() + public async buildMapAssets(data: { game_server_node_id: string }) { + return { + success: await this.mapAssets.queueManualBuild(data.game_server_node_id), + }; + } +} diff --git a/src/map-assets/map-assets.module.ts b/src/map-assets/map-assets.module.ts new file mode 100644 index 000000000..8a78907b6 --- /dev/null +++ b/src/map-assets/map-assets.module.ts @@ -0,0 +1,35 @@ +import { Module } from "@nestjs/common"; +import { BullModule } from "@nestjs/bullmq"; +import { BullBoardModule } from "@bull-board/nestjs"; +import { BullMQAdapter } from "@bull-board/api/bullMQAdapter"; +import { PostgresModule } from "../postgres/postgres.module"; +import { K8sModule } from "../k8s/k8s.module"; +import { getQueuesProcessors } from "../utilities/QueueProcessors"; +import { loggerFactory } from "../utilities/LoggerFactory"; +import { MapAssetsQueues } from "./enums/MapAssetsQueues"; +import { MapAssetsService } from "./map-assets.service"; +import { MapAssetsController } from "./map-assets.controller"; +import { BuildMapAssets } from "./jobs/BuildMapAssets"; + +@Module({ + imports: [ + PostgresModule, + K8sModule, + BullModule.registerQueue({ + name: MapAssetsQueues.BuildMapAssets, + }), + BullBoardModule.forFeature({ + name: MapAssetsQueues.BuildMapAssets, + adapter: BullMQAdapter, + }), + ], + controllers: [MapAssetsController], + providers: [ + MapAssetsService, + BuildMapAssets, + ...getQueuesProcessors("MapAssets"), + loggerFactory(), + ], + exports: [MapAssetsService], +}) +export class MapAssetsModule {} diff --git a/src/map-assets/map-assets.service.spec.ts b/src/map-assets/map-assets.service.spec.ts new file mode 100644 index 000000000..2193ae935 --- /dev/null +++ b/src/map-assets/map-assets.service.spec.ts @@ -0,0 +1,552 @@ +import { MapAssetsService } from "./map-assets.service"; + +describe("MapAssetsService", () => { + const originalDomain = process.env.WEB_DOMAIN; + const originalFetch = global.fetch; + + type Db = { + autoBuild: string | null; + current: boolean; + claimed: boolean; + nodeBuild: number | null; + existing: string | null; + }; + + let db: Db; + let postgres: { query: jest.Mock }; + let loggingService: { getJobStatus: jest.Mock; getJobPod: jest.Mock }; + let queue: { add: jest.Mock; getJob: jest.Mock }; + let batchApi: { + deleteNamespacedJob: jest.Mock; + createNamespacedJob: jest.Mock; + }; + let coreApi: { readNamespacedPodLog: jest.Mock }; + let files: Record; + let service: MapAssetsService; + + beforeEach(() => { + process.env.WEB_DOMAIN = "5stack.gg"; + + db = { + autoBuild: "true", + current: true, + claimed: true, + nodeBuild: 25537370, + existing: null, + }; + postgres = { + query: jest.fn(async (sql: string, params: Array) => { + if (sql.includes("FROM public.settings")) { + return db.autoBuild === null ? [] : [{ value: db.autoBuild }]; + } + if (sql.includes("FROM public.game_versions")) { + return db.current ? [{ build_id: params[0] }] : []; + } + if (sql.includes("DO NOTHING")) { + return db.claimed ? [{ build_id: params[0] }] : []; + } + if (sql.includes("FROM public.game_server_nodes")) { + return [{ build_id: db.nodeBuild }]; + } + if (sql.includes("SELECT status")) { + return db.existing ? [{ status: db.existing }] : []; + } + return []; + }), + }; + loggingService = { + getJobStatus: jest.fn(), + getJobPod: jest.fn().mockResolvedValue(undefined), + }; + queue = { + add: jest.fn().mockResolvedValue({}), + getJob: jest.fn().mockResolvedValue(undefined), + }; + batchApi = { + deleteNamespacedJob: jest.fn().mockResolvedValue({}), + createNamespacedJob: jest.fn().mockResolvedValue({}), + }; + coreApi = { readNamespacedPodLog: jest.fn().mockResolvedValue("") }; + + files = {}; + global.fetch = jest.fn(async (url: string) => { + const key = url.replace("https://demo-dl.5stack.gg/maps/", ""); + if (!(key in files)) { + return { + ok: false, + status: 404, + json: async (): Promise => null, + }; + } + return { ok: true, status: 200, json: async () => files[key] }; + }) as any; + + service = new MapAssetsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + { get: () => ({ namespace: "5stack" }) } as any, + postgres as any, + loggingService as any, + queue as any, + ); + (service as any).batchApi = batchApi; + (service as any).coreApi = coreApi; + }); + + afterEach(() => { + process.env.WEB_DOMAIN = originalDomain; + global.fetch = originalFetch; + }); + + const sqlCalls = () => + postgres.query.mock.calls.map(([sql]) => String(sql).replace(/\s+/g, " ")); + + describe("jobSpec", () => { + const job = MapAssetsService.jobSpec("node.one", "25537370"); + const pod = job.spec.template.spec; + const container = pod.containers[0]; + + it("is named for the build so a second run replaces the first", () => { + expect(job.metadata.name).toBe("map-assets-25537370"); + }); + + it("is pinned to the node whose install it reads", () => { + expect( + pod.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution + .nodeSelectorTerms[0].matchExpressions[0], + ).toEqual({ + key: "kubernetes.io/hostname", + operator: "In", + values: ["node.one"], + }); + }); + + it("mounts the node's install read-only and scratch space for the build", () => { + expect(container.volumeMounts).toEqual([ + { + name: "serverfiles-node-one", + mountPath: "/serverdata/serverfiles", + readOnly: true, + }, + { name: "work", mountPath: "/work" }, + { name: "tmp", mountPath: "/tmp" }, + ]); + expect(pod.volumes).toEqual([ + { + name: "serverfiles-node-one", + persistentVolumeClaim: { + claimName: "serverfiles-node-one-claim", + readOnly: true, + }, + }, + { name: "work", emptyDir: { sizeLimit: "4Gi" } }, + { name: "tmp", emptyDir: { sizeLimit: "8Gi" } }, + ]); + }); + + it("runs the publisher against that install, writing into /work", () => { + expect(container.image).toBe("ghcr.io/5stackgg/map-assets:latest"); + expect(container.args).toEqual([ + "--cs2", + "/serverdata/serverfiles", + "--build", + "25537370", + "--out", + "/work", + "--publish", + ]); + }); + + it("takes the B2 credentials from s3-secrets", () => { + expect(container.env).toEqual([ + { + name: "S3_ACCESS_KEY", + valueFrom: { + secretKeyRef: { name: "s3-secrets", key: "S3_ACCESS_KEY" }, + }, + }, + { + name: "S3_SECRET", + valueFrom: { secretKeyRef: { name: "s3-secrets", key: "S3_SECRET" } }, + }, + ]); + }); + + it("never retries, is killed at the timeout and is kept a week", () => { + expect(pod.restartPolicy).toBe("Never"); + expect(job.spec.backoffLimit).toBe(0); + expect(job.spec.activeDeadlineSeconds).toBe(2 * 60 * 60); + expect(job.spec.ttlSecondsAfterFinished).toBe(60 * 60 * 24 * 7); + }); + + it("is sized for Source2Viewer but bounded so the node's game servers keep running", () => { + expect(container.resources).toEqual({ + requests: { cpu: "1", memory: "4Gi", "ephemeral-storage": "6Gi" }, + limits: { cpu: "2", memory: "8Gi", "ephemeral-storage": "14Gi" }, + }); + }); + }); + + describe("queueBuild", () => { + it("does nothing off the public instance", async () => { + process.env.WEB_DOMAIN = "example.com"; + + await expect(service.queueBuild("node-1", 25537370)).resolves.toBe(false); + expect(postgres.query).not.toHaveBeenCalled(); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it.each([null, "false"])( + "stays off until map_assets_auto_build is turned on (%s)", + async (value) => { + db.autoBuild = value; + + await expect(service.queueBuild("node-1", 25537370)).resolves.toBe( + false, + ); + expect(postgres.query.mock.calls[0][1]).toEqual([ + "map_assets_auto_build", + ]); + expect(sqlCalls()).toHaveLength(1); + expect(queue.add).not.toHaveBeenCalled(); + }, + ); + + it("ignores a build that is not the current one", async () => { + db.current = false; + + await expect(service.queueBuild("node-1", 25537370)).resolves.toBe(false); + expect(sqlCalls().some((sql) => sql.includes("INSERT"))).toBe(false); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("does not queue a build that already has a row", async () => { + db.claimed = false; + + await expect(service.queueBuild("node-1", 25537370)).resolves.toBe(false); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("claims the build and queues one job per build", async () => { + await expect(service.queueBuild("node-1", 25537370)).resolves.toBe(true); + expect(queue.add).toHaveBeenCalledWith( + "BuildMapAssets", + { gameServerNodeId: "node-1", buildId: "25537370" }, + expect.objectContaining({ + jobId: "map-assets.25537370", + attempts: 1, + }), + ); + }); + + it("releases the claim when the job cannot be queued", async () => { + queue.add.mockRejectedValueOnce(new Error("redis down")); + + await expect(service.queueBuild("node-1", 25537370)).rejects.toThrow( + "redis down", + ); + const release = sqlCalls().at(-1); + expect(release).toContain("DELETE FROM public.map_asset_builds"); + expect(release).toContain("status = 'Pending'"); + }); + }); + + describe("queueManualBuild", () => { + it("does nothing off the public instance", async () => { + process.env.WEB_DOMAIN = "example.com"; + + await expect(service.queueManualBuild("node-1")).resolves.toBe(false); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("ignores the automatic-build setting", async () => { + db.autoBuild = null; + + await expect(service.queueManualBuild("node-1")).resolves.toBe(true); + expect(sqlCalls().some((sql) => sql.includes("public.settings"))).toBe( + false, + ); + }); + + it("needs the node to have reported a build", async () => { + db.nodeBuild = null; + + await expect(service.queueManualBuild("node-1")).resolves.toBe(false); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it.each(["Partial", "Failed", "Pending"])( + "retries a %s build", + async (status) => { + db.existing = status; + + await expect(service.queueManualBuild("node-1")).resolves.toBe(true); + expect(sqlCalls().at(-1)).toContain("SET status = 'Pending'"); + expect(queue.add).toHaveBeenCalledWith( + "BuildMapAssets", + { gameServerNodeId: "node-1", buildId: "25537370" }, + expect.objectContaining({ jobId: "map-assets.25537370" }), + ); + }, + ); + + it("leaves a published build alone", async () => { + db.existing = "Published"; + + await expect(service.queueManualBuild("node-1")).resolves.toBe(false); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("does not start a second run while one is queued or running", async () => { + db.existing = "Building"; + queue.getJob.mockResolvedValueOnce({ id: "map-assets.25537370" }); + + await expect(service.queueManualBuild("node-1")).resolves.toBe(false); + expect(queue.getJob).toHaveBeenCalledWith("map-assets.25537370"); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("recovers a Building row whose job is gone", async () => { + db.existing = "Building"; + + await expect(service.queueManualBuild("node-1")).resolves.toBe(true); + }); + }); + + describe("build", () => { + const maps = { + de_mirage: { + tri: "25537370/de_mirage.tri.gz", + callouts: "25000000/de_mirage.callouts.json", + }, + }; + + const finalUpdate = () => { + const call = postgres.query.mock.calls.at(-1); + return { sql: String(call[0]), params: call[1] }; + }; + + const finished = (status: Record) => { + loggingService.getJobStatus + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(status); + }; + + const exited = (exitCode: number) => { + loggingService.getJobPod.mockResolvedValue({ + metadata: { name: "map-assets-25537370-abcde" }, + status: { + containerStatuses: [ + { + name: "map-assets", + state: { terminated: { exitCode, reason: "Error" } }, + }, + ], + }, + }); + }; + + it("reads the manifest revision latest.json names for this build", async () => { + finished({ succeeded: 1 }); + files["latest.json"] = { + version: 1, + build: "25537370", + manifest: "25537370/manifest.r2.json", + }; + files["25537370/manifest.r2.json"] = { + version: 1, + build: "25537370", + maps, + }; + + const outcome = await service.build("node-1", "25537370"); + + expect(batchApi.createNamespacedJob).toHaveBeenCalledWith({ + namespace: "5stack", + body: MapAssetsService.jobSpec("node-1", "25537370"), + }); + expect(outcome).toEqual({ + status: "Published", + manifest: "25537370/manifest.r2.json", + maps, + failed: [], + failed_view: [], + error: null, + }); + expect(finalUpdate().params).toEqual([ + "25537370", + "Published", + "25537370/manifest.r2.json", + JSON.stringify(maps), + "[]", + "[]", + null, + ]); + }); + + it("reads the build's first manifest when latest.json names another build", async () => { + finished({ succeeded: 1 }); + files["latest.json"] = { + version: 1, + build: "25600000", + manifest: "25600000/manifest.json", + }; + files["25537370/manifest.json"] = { version: 1, build: "25537370", maps }; + + const outcome = await service.build("node-1", "25537370"); + + expect(outcome.manifest).toBe("25537370/manifest.json"); + expect(outcome.maps).toEqual(maps); + }); + + it("records a partial publish with the maps that failed", async () => { + finished({ + failed: 1, + conditions: [ + { type: "Failed", status: "True", reason: "BackoffLimitExceeded" }, + ], + }); + exited(2); + coreApi.readNamespacedPodLog.mockResolvedValue( + "de_anubis: Source2Viewer-CLI exited 134\n", + ); + files["latest.json"] = { + version: 1, + build: "25537370", + manifest: "25537370/manifest.json", + }; + files["25537370/manifest.json"] = { + version: 1, + build: "25537370", + maps, + failed: ["de_anubis"], + failed_view: ["de_vertigo"], + }; + + const outcome = await service.build("node-1", "25537370"); + + expect(outcome).toMatchObject({ + status: "Partial", + manifest: "25537370/manifest.json", + maps, + failed: ["de_anubis"], + failed_view: ["de_vertigo"], + }); + expect(outcome.error).toContain("Source2Viewer-CLI exited 134"); + expect(finalUpdate().params[1]).toBe("Partial"); + }); + + it("records why a failed job failed", async () => { + finished({ + failed: 1, + conditions: [ + { + type: "Failed", + status: "True", + reason: "BackoffLimitExceeded", + message: "Job has reached the specified backoff limit", + }, + ], + }); + exited(1); + coreApi.readNamespacedPodLog.mockResolvedValue( + "extracting de_mirage\nupload refused\n", + ); + + const outcome = await service.build("node-1", "25537370"); + + expect(outcome.status).toBe("Failed"); + expect(outcome.error).toContain("BackoffLimitExceeded"); + expect(outcome.error).toContain("Error (exit 1)"); + expect(outcome.error).toContain("upload refused"); + expect(global.fetch).not.toHaveBeenCalled(); + const update = finalUpdate(); + expect(update.params[1]).toBe("Failed"); + expect(update.sql).toContain("maps = COALESCE($4::jsonb, maps)"); + }); + + it("attaches to a job that is still running instead of restarting it", async () => { + loggingService.getJobStatus + .mockResolvedValueOnce({ active: 1 }) + .mockResolvedValueOnce({ succeeded: 1 }); + + const outcome = await service.build("node-1", "25537370"); + + expect(batchApi.deleteNamespacedJob).not.toHaveBeenCalled(); + expect(batchApi.createNamespacedJob).not.toHaveBeenCalled(); + expect(outcome.status).toBe("Published"); + }); + + it("marks the build failed when the job cannot be created", async () => { + loggingService.getJobStatus.mockResolvedValueOnce(undefined); + batchApi.createNamespacedJob.mockRejectedValueOnce( + new Error("forbidden"), + ); + + const outcome = await service.build("node-1", "25537370"); + + expect(outcome).toMatchObject({ status: "Failed", error: "forbidden" }); + expect(finalUpdate().sql).toContain("UPDATE public.map_asset_builds"); + }); + }); + + describe("assetUrl", () => { + const manifest = { + version: 1, + build: "25537370", + maps: { + de_mirage: { + tri: "25537370/de_mirage.tri.gz", + callouts: "25000000/de_mirage.callouts.json", + view: "25537370/de_mirage.view.bin.gz", + }, + de_nuke: { callouts: "25537370/de_nuke.callouts.json" }, + de_bad: { callouts: "../../secret.json", tri: "/etc/passwd" }, + }, + }; + const pinned = "https://demo-dl.5stack.gg/maps/24957633"; + + it("resolves a manifest key under the maps root", () => { + expect(MapAssetsService.assetUrl(manifest, "de_mirage", "callouts")).toBe( + "https://demo-dl.5stack.gg/maps/25000000/de_mirage.callouts.json", + ); + }); + + it("falls back per asset, so an entry without a tri keeps the pinned tri", () => { + expect(MapAssetsService.assetUrl(manifest, "de_nuke", "tri")).toBe( + `${pinned}/de_nuke.tri.gz`, + ); + expect(MapAssetsService.assetUrl(manifest, "de_nuke", "callouts")).toBe( + "https://demo-dl.5stack.gg/maps/25537370/de_nuke.callouts.json", + ); + }); + + it("falls back to the pinned build for a map the manifest does not list", () => { + expect(MapAssetsService.assetUrl(manifest, "de_dust2", "callouts")).toBe( + `${pinned}/de_dust2.callouts.json`, + ); + expect(MapAssetsService.assetUrl(null, "de_dust2", "tri")).toBe( + `${pinned}/de_dust2.tri.gz`, + ); + expect( + MapAssetsService.assetUrl(manifest, "constructor", "callouts"), + ).toBe(`${pinned}/constructor.callouts.json`); + }); + + it("never falls back for assets the pinned build never had", () => { + expect(MapAssetsService.assetUrl(manifest, "de_nuke", "view")).toBe(null); + expect( + MapAssetsService.assetUrl(manifest, "de_mirage", "grenadeclip"), + ).toBe(null); + }); + + it("treats a key that would leave the maps root as unlisted", () => { + expect(MapAssetsService.assetUrl(manifest, "de_bad", "callouts")).toBe( + `${pinned}/de_bad.callouts.json`, + ); + expect(MapAssetsService.assetUrl(manifest, "de_bad", "tri")).toBe( + `${pinned}/de_bad.tri.gz`, + ); + expect(MapAssetsService.assetUrl(manifest, "../x", "tri")).toBe(null); + }); + }); +}); diff --git a/src/map-assets/map-assets.service.ts b/src/map-assets/map-assets.service.ts new file mode 100644 index 000000000..65728247d --- /dev/null +++ b/src/map-assets/map-assets.service.ts @@ -0,0 +1,722 @@ +import { Injectable, Logger } from "@nestjs/common"; +import { ConfigService } from "@nestjs/config"; +import { InjectQueue } from "@nestjs/bullmq"; +import { Queue } from "bullmq"; +import { + BatchV1Api, + CoreV1Api, + KubeConfig, + V1Job, + V1JobStatus, + V1Pod, +} from "@kubernetes/client-node"; +import { PostgresService } from "../postgres/postgres.service"; +import { LoggingService } from "../k8s/logging/logging.service"; +import { GameServersConfig } from "../configs/types/GameServersConfig"; +import { SystemSettingName } from "../system/enums/SystemSettingName"; +import { MapAssetsQueues } from "./enums/MapAssetsQueues"; + +export type MapAssetKind = "tri" | "grenadeclip" | "view" | "callouts"; + +export type MapAssetsManifest = { + version: number; + build: string; + created_at?: string; + maps: Record>>; + failed?: Array; + failed_view?: Array; +}; + +export type MapAssetsPointer = { + version: number; + build: string; + manifest: string; +}; + +export type MapAssetBuildStatus = + | "Pending" + | "Building" + | "Published" + | "Partial" + | "Failed"; + +export type MapAssetBuildOutcome = { + status: "Published" | "Partial" | "Failed"; + manifest: string | null; + maps: MapAssetsManifest["maps"] | null; + failed: Array | null; + failed_view: Array | null; + error: string | null; +}; + +@Injectable() +export class MapAssetsService { + public static readonly MAPS_HOST = "https://demo-dl.5stack.gg/maps"; + + // The last build published before manifests existed. Every consumer falls + // back to its flat /.* files when latest.json cannot be read, and + // per asset when the manifest has no entry for it. + public static readonly PINNED_BUILD = "24957633"; + + // Only the assets the pinned build actually carried; views and grenade clips + // came with manifests, so an unlisted one is simply absent. + private static readonly PINNED_FILES: Partial> = + { + tri: ".tri.gz", + callouts: ".callouts.json", + }; + + public static readonly INDEX_VERSION = 1; + + public static readonly IMAGE = "ghcr.io/5stackgg/map-assets:latest"; + + public static readonly TIMEOUT_MS = 2 * 60 * 60 * 1000; + + // The publisher exits 2 when it published a manifest but some maps failed + // (listed in the manifest's failed / failed_view); anything else non-zero + // means nothing usable was published. + private static readonly PARTIAL_EXIT_CODE = 2; + + private static readonly POLL_MS = 15_000; + + private static readonly LOG_TAIL_LINES = 40; + + private static readonly MAX_LOG_CHARS = 3000; + + private readonly namespace: string; + private readonly coreApi: CoreV1Api; + private readonly batchApi: BatchV1Api; + + constructor( + private readonly logger: Logger, + private readonly config: ConfigService, + private readonly postgres: PostgresService, + private readonly loggingService: LoggingService, + @InjectQueue(MapAssetsQueues.BuildMapAssets) + private readonly queue: Queue, + ) { + this.namespace = + this.config.get("gameServers").namespace; + + const kc = new KubeConfig(); + kc.loadFromDefault(); + + this.coreApi = kc.makeApiClient(CoreV1Api); + this.batchApi = kc.makeApiClient(BatchV1Api); + } + + public static GET_JOB_NAME(buildId: string) { + return `map-assets-${buildId}`; + } + + public static GET_QUEUE_JOB_ID(buildId: string) { + return `map-assets.${buildId}`; + } + + public static isSafeKey(key: unknown): key is string { + if (typeof key !== "string" || key.length === 0) { + return false; + } + + return key + .split("/") + .every( + (part) => + part !== "." && part !== ".." && /^[A-Za-z0-9._-]+$/.test(part), + ); + } + + // Per asset, not per map: an entry that exists but lacks this asset (or names + // an unusable key) falls back exactly as a missing entry does. + public static assetUrl( + manifest: MapAssetsManifest | null, + map: string, + kind: MapAssetKind, + ): string | null { + const entry = + manifest?.maps && Object.hasOwn(manifest.maps, map) + ? manifest.maps[map] + : null; + const key = entry?.[kind]; + + if (MapAssetsService.isSafeKey(key)) { + return `${MapAssetsService.MAPS_HOST}/${key}`; + } + + const suffix = MapAssetsService.PINNED_FILES[kind]; + if (!suffix || !MapAssetsService.isSafeKey(map)) { + return null; + } + + return `${MapAssetsService.MAPS_HOST}/${MapAssetsService.PINNED_BUILD}/${map}${suffix}`; + } + + // B2 has no ListBucket grant on these keys, so a missing object can arrive as + // 403 as well as 404; both mean "not published". + public static async fetchIndex(key: string): Promise { + const response = await fetch(`${MapAssetsService.MAPS_HOST}/${key}`, { + signal: AbortSignal.timeout(15_000), + }); + + if (response.status === 404 || response.status === 403) { + return null; + } + + if (!response.ok) { + throw new Error(`${key} answered ${response.status}`); + } + + return (await response.json()) as T; + } + + public static jobSpec(gameServerNodeId: string, buildId: string): V1Job { + const volume = `serverfiles-${gameServerNodeId.replaceAll(".", "-")}`; + + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: MapAssetsService.GET_JOB_NAME(buildId), + labels: { + app: "map-assets", + }, + }, + spec: { + backoffLimit: 0, + activeDeadlineSeconds: MapAssetsService.TIMEOUT_MS / 1000, + ttlSecondsAfterFinished: 60 * 60 * 24 * 7, + template: { + metadata: { + labels: { + app: "map-assets", + }, + }, + spec: { + affinity: { + nodeAffinity: { + requiredDuringSchedulingIgnoredDuringExecution: { + nodeSelectorTerms: [ + { + matchExpressions: [ + { + key: "kubernetes.io/hostname", + operator: "In", + values: [gameServerNodeId], + }, + ], + }, + ], + }, + }, + }, + restartPolicy: "Never", + dnsConfig: { + options: [ + { + name: "ndots", + value: "1", + }, + ], + }, + containers: [ + { + name: "map-assets", + image: MapAssetsService.IMAGE, + args: [ + "--cs2", + "/serverdata/serverfiles", + "--build", + buildId, + "--out", + "/work", + "--publish", + ], + env: [ + { + name: "S3_ACCESS_KEY", + valueFrom: { + secretKeyRef: { + name: "s3-secrets", + key: "S3_ACCESS_KEY", + }, + }, + }, + { + name: "S3_SECRET", + valueFrom: { + secretKeyRef: { + name: "s3-secrets", + key: "S3_SECRET", + }, + }, + }, + ], + volumeMounts: [ + { + name: volume, + mountPath: "/serverdata/serverfiles", + readOnly: true, + }, + { + name: "work", + mountPath: "/work", + }, + { + name: "tmp", + mountPath: "/tmp", + }, + ], + // Source2Viewer peaks around 5 GB on rush_001, and the exports + // it writes before simplification are several GB on disk. + resources: { + requests: { + cpu: "1", + memory: "4Gi", + "ephemeral-storage": "6Gi", + }, + limits: { + cpu: "2", + memory: "8Gi", + "ephemeral-storage": "14Gi", + }, + }, + }, + ], + volumes: [ + { + name: volume, + persistentVolumeClaim: { + claimName: `${volume}-claim`, + readOnly: true, + }, + }, + { + name: "work", + emptyDir: { + sizeLimit: "4Gi", + }, + }, + { + name: "tmp", + emptyDir: { + sizeLimit: "8Gi", + }, + }, + ], + }, + }, + }, + }; + } + + // The automatic path, gated on the public instance and on an operator having + // turned it on. A build is claimed by inserting its row, so every node + // reporting the same new build races for one INSERT rather than queueing a + // job each. + public async queueBuild( + gameServerNodeId: string, + buildId: number, + ): Promise { + if (process.env.WEB_DOMAIN !== "5stack.gg") { + return false; + } + + if (!(await this.autoBuildEnabled())) { + return false; + } + + const [current] = await this.postgres.query>( + `SELECT build_id + FROM public.game_versions + WHERE current = true + AND build_id = $1`, + [buildId], + ); + + if (!current) { + return false; + } + + const claimed = await this.postgres.query>( + `INSERT INTO public.map_asset_builds (build_id) + VALUES ($1) + ON CONFLICT (build_id) DO NOTHING + RETURNING build_id`, + [String(buildId)], + ); + + if (!claimed.length) { + return false; + } + + try { + await this.enqueue(gameServerNodeId, String(buildId)); + } catch (error) { + await this.postgres.query( + `DELETE FROM public.map_asset_builds + WHERE build_id = $1 + AND status = 'Pending'`, + [String(buildId)], + ); + throw error; + } + + return true; + } + + // Retries a Partial or Failed build (the publisher rebuilds only the failed + // maps into a new manifest revision). A Published build is immutable, and one + // with a live queue job is already running; a Building row whose job is gone + // is what an api crash leaves behind, so it may be re-run. + public async queueManualBuild(gameServerNodeId: string): Promise { + if (process.env.WEB_DOMAIN !== "5stack.gg") { + return false; + } + + const [node] = await this.postgres.query< + Array<{ build_id: number | null }> + >( + `SELECT build_id + FROM public.game_server_nodes + WHERE id = $1`, + [gameServerNodeId], + ); + + if (!node?.build_id) { + return false; + } + + const buildId = String(node.build_id); + + const [existing] = await this.postgres.query< + Array<{ status: MapAssetBuildStatus }> + >( + `SELECT status + FROM public.map_asset_builds + WHERE build_id = $1`, + [buildId], + ); + + if (existing?.status === "Published") { + return false; + } + + if (await this.queue.getJob(MapAssetsService.GET_QUEUE_JOB_ID(buildId))) { + return false; + } + + await this.postgres.query( + `INSERT INTO public.map_asset_builds (build_id) + VALUES ($1) + ON CONFLICT (build_id) DO UPDATE + SET status = 'Pending', + error = NULL`, + [buildId], + ); + + await this.enqueue(gameServerNodeId, buildId); + + return true; + } + + // A job that is still running is attached to rather than replaced: the api + // restarting mid-build hands the BullMQ job back as stalled, and killing an + // hour of extraction to start it again would be the wrong answer. + public async build( + gameServerNodeId: string, + buildId: string, + ): Promise { + const jobName = MapAssetsService.GET_JOB_NAME(buildId); + + await this.postgres.query( + `INSERT INTO public.map_asset_builds (build_id, status, started_at) + VALUES ($1, 'Building', now()) + ON CONFLICT (build_id) DO UPDATE + SET status = 'Building', + started_at = CASE + WHEN map_asset_builds.status = 'Building' + THEN map_asset_builds.started_at + ELSE now() + END, + finished_at = NULL, + error = NULL`, + [buildId], + ); + + let outcome: MapAssetBuildOutcome; + + try { + const existing = await this.loggingService.getJobStatus(jobName); + + if (!existing?.active || existing.succeeded || existing.failed) { + await this.startJob(gameServerNodeId, buildId); + } + + outcome = await this.waitForJob(jobName, buildId); + } catch (error) { + outcome = MapAssetsService.failed( + (error as Error)?.message ?? String(error), + ); + } + + await this.postgres.query( + `UPDATE public.map_asset_builds + SET status = $2, + finished_at = now(), + manifest = COALESCE($3, manifest), + maps = COALESCE($4::jsonb, maps), + failed = COALESCE($5::jsonb, failed), + failed_view = COALESCE($6::jsonb, failed_view), + error = $7 + WHERE build_id = $1`, + [ + buildId, + outcome.status, + outcome.manifest, + outcome.maps ? JSON.stringify(outcome.maps) : null, + outcome.failed ? JSON.stringify(outcome.failed) : null, + outcome.failed_view ? JSON.stringify(outcome.failed_view) : null, + outcome.error, + ], + ); + + return outcome; + } + + private static failed(error: string): MapAssetBuildOutcome { + return { + status: "Failed", + manifest: null, + maps: null, + failed: null, + failed_view: null, + error, + }; + } + + private async autoBuildEnabled(): Promise { + const [setting] = await this.postgres.query>( + `SELECT value FROM public.settings WHERE name = $1 LIMIT 1`, + [SystemSettingName.MapAssetsAutoBuild], + ); + + return setting?.value === "true"; + } + + private async enqueue(gameServerNodeId: string, buildId: string) { + await this.queue.add( + "BuildMapAssets", + { + gameServerNodeId, + buildId, + }, + { + jobId: MapAssetsService.GET_QUEUE_JOB_ID(buildId), + attempts: 1, + removeOnComplete: true, + removeOnFail: true, + }, + ); + } + + private async startJob(gameServerNodeId: string, buildId: string) { + await this.batchApi + .deleteNamespacedJob({ + name: MapAssetsService.GET_JOB_NAME(buildId), + namespace: this.namespace, + propagationPolicy: "Background", + gracePeriodSeconds: 0, + }) + .catch((error) => { + if (error.code?.toString() !== "404") { + throw error; + } + }); + + await this.batchApi.createNamespacedJob({ + namespace: this.namespace, + body: MapAssetsService.jobSpec(gameServerNodeId, buildId), + }); + } + + private async waitForJob( + jobName: string, + buildId: string, + ): Promise { + const deadline = Date.now() + MapAssetsService.TIMEOUT_MS + 5 * 60 * 1000; + + let status: V1JobStatus | undefined; + + while (Date.now() < deadline) { + status = await this.loggingService.getJobStatus(jobName); + if (!status || status.succeeded || status.failed) { + break; + } + await new Promise((resolve) => + setTimeout(resolve, MapAssetsService.POLL_MS), + ); + } + + if (status?.succeeded) { + return { + ...(await this.publishedManifest(buildId)), + status: "Published", + error: null, + }; + } + + const pod = status?.failed + ? await this.loggingService.getJobPod(jobName) + : undefined; + + if (MapAssetsService.exitCode(pod) === MapAssetsService.PARTIAL_EXIT_CODE) { + return { + ...(await this.publishedManifest(buildId)), + status: "Partial", + error: (await this.logTail(pod)) ?? null, + }; + } + + return MapAssetsService.failed( + await this.failureReason(jobName, status, pod), + ); + } + + // A rebuild of a Partial build publishes a new manifest revision + // (/manifest.rN.json) and repoints latest.json at it, so latest.json + // is the authority whenever it names this build. + private async publishedManifest( + buildId: string, + ): Promise< + Pick + > { + try { + const pointer = + await MapAssetsService.fetchIndex("latest.json"); + const key = + pointer?.build === buildId && + MapAssetsService.isSafeKey(pointer.manifest) + ? pointer.manifest + : `${buildId}/manifest.json`; + const manifest = + await MapAssetsService.fetchIndex(key); + + if (manifest) { + return { + manifest: key, + maps: manifest.maps ?? null, + failed: manifest.failed ?? [], + failed_view: manifest.failed_view ?? [], + }; + } + + this.logger.warn( + `[map-assets] build ${buildId} finished but ${key} is not published`, + ); + } catch (error) { + this.logger.warn( + `[map-assets] build ${buildId} finished but its manifest could not be read: ${(error as Error)?.message}`, + ); + } + + return { manifest: null, maps: null, failed: null, failed_view: null }; + } + + private static exitCode(pod: V1Pod | undefined): number | null { + const terminated = pod?.status?.containerStatuses?.find( + (containerStatus) => containerStatus.name === "map-assets", + )?.state?.terminated; + + return terminated ? terminated.exitCode : null; + } + + private async failureReason( + jobName: string, + status: V1JobStatus | undefined, + pod: V1Pod | undefined, + ): Promise { + const reasons: Array = []; + + const failed = status?.conditions?.find( + (condition) => condition.type === "Failed" && condition.status === "True", + ); + + if (failed) { + reasons.push([failed.reason, failed.message].filter(Boolean).join(": ")); + } else if (!status) { + reasons.push(`${jobName} disappeared before it finished`); + } else { + reasons.push( + `${jobName} did not finish within ${MapAssetsService.TIMEOUT_MS / 3_600_000}h`, + ); + } + + const jobPod = pod ?? (await this.loggingService.getJobPod(jobName)); + + const podReason = MapAssetsService.podFailureReason(jobPod); + if (podReason) { + reasons.push(podReason); + } + + const tail = await this.logTail(jobPod); + if (tail) { + reasons.push(tail); + } + + return reasons.filter(Boolean).join("\n"); + } + + private async logTail(pod: V1Pod | undefined): Promise { + if (!pod?.metadata?.name) { + return null; + } + + try { + const logs = await this.coreApi.readNamespacedPodLog({ + name: pod.metadata.name, + namespace: this.namespace, + tailLines: MapAssetsService.LOG_TAIL_LINES, + }); + const tail = String(logs ?? "").trim(); + return tail ? tail.slice(-MapAssetsService.MAX_LOG_CHARS) : null; + } catch { + this.logger.warn(`[map-assets] ${pod.metadata.name} produced no logs`); + return null; + } + } + + private static podFailureReason(pod: V1Pod | undefined): string | null { + if (!pod) { + return null; + } + + const statuses = [ + ...(pod.status?.initContainerStatuses ?? []), + ...(pod.status?.containerStatuses ?? []), + ]; + + for (const containerStatus of statuses) { + const waiting = containerStatus.state?.waiting; + if (waiting?.reason) { + return waiting.message + ? `${waiting.reason}: ${waiting.message}` + : waiting.reason; + } + + const terminated = containerStatus.state?.terminated; + if (terminated && terminated.exitCode !== 0) { + return `${terminated.reason ?? "terminated"} (exit ${terminated.exitCode})`; + } + } + + const unscheduled = pod.status?.conditions?.find( + (condition) => + condition.type === "PodScheduled" && condition.status === "False", + ); + if (unscheduled) { + return unscheduled.message ?? unscheduled.reason ?? "Unschedulable"; + } + + return pod.status?.message ?? null; + } +} diff --git a/src/system/enums/SystemSettingName.ts b/src/system/enums/SystemSettingName.ts index 9c94fb3e9..25ea3dbd4 100644 --- a/src/system/enums/SystemSettingName.ts +++ b/src/system/enums/SystemSettingName.ts @@ -76,4 +76,8 @@ export enum SystemSettingName { // is never exposed to any role; see public_settings.yaml. WebPushPublicKey = "web_push_public_key", WebPushPrivateKey = "web_push_private_key", + // Off until the map-assets worker is deployed: a build published before the + // CDN can serve it would point latest.json at files nothing can read. Gates + // the build a new CS2 version triggers, never the manual action. + MapAssetsAutoBuild = "map_assets_auto_build", } diff --git a/src/utilities/QueueProcessors.ts b/src/utilities/QueueProcessors.ts index 57c35070e..173dde998 100644 --- a/src/utilities/QueueProcessors.ts +++ b/src/utilities/QueueProcessors.ts @@ -34,7 +34,8 @@ type Modules = | "Scrims" | "Chat" | "Utility" - | "Voice"; + | "Voice" + | "MapAssets"; export type UseQueueOptions = { concurrency?: number; diff --git a/src/utility/utility-callouts.service.spec.ts b/src/utility/utility-callouts.service.spec.ts index cb3270b3f..95204bd09 100644 --- a/src/utility/utility-callouts.service.spec.ts +++ b/src/utility/utility-callouts.service.spec.ts @@ -146,3 +146,170 @@ describe("auto naming", () => { ).resolves.toBe(""); }); }); + +describe("UtilityCalloutsService.calloutsUrl", () => { + const originalCdn = process.env.MAP_MESH_CDN; + const originalFetch = global.fetch; + + const pointer = { + version: 1, + build: "25537370", + manifest: "25537370/manifest.json", + }; + const manifest = { + version: 1, + build: "25537370", + maps: { + de_mirage: { callouts: "25000000/de_mirage.callouts.json" }, + de_nuke: { callouts: "25537370/de_nuke.callouts.json" }, + de_vertigo: { tri: "25537370/de_vertigo.tri.gz" }, + }, + }; + + const respond = (body: unknown, status = 200) => ({ + ok: status >= 200 && status < 300, + status, + json: async () => body, + }); + + let files: Record; + let fetchMock: jest.Mock; + let service: UtilityCalloutsService; + + const fetched = (key: string) => + fetchMock.mock.calls.filter( + ([url]) => url === `https://demo-dl.5stack.gg/maps/${key}`, + ).length; + + beforeEach(() => { + delete process.env.MAP_MESH_CDN; + files = { + "latest.json": { body: pointer, status: 200 }, + "25537370/manifest.json": { body: manifest, status: 200 }, + }; + fetchMock = jest.fn(async (url: string) => { + const key = url.replace("https://demo-dl.5stack.gg/maps/", ""); + const file = files[key] ?? { status: 404 }; + return respond(file.body, file.status); + }); + global.fetch = fetchMock as any; + service = new UtilityCalloutsService( + { warn: jest.fn() } as never, + null as never, + ); + }); + + afterEach(() => { + if (originalCdn === undefined) { + delete process.env.MAP_MESH_CDN; + } else { + process.env.MAP_MESH_CDN = originalCdn; + } + global.fetch = originalFetch; + jest.restoreAllMocks(); + }); + + it("keeps MAP_MESH_CDN as a flat directory", async () => { + process.env.MAP_MESH_CDN = "https://mirror.test/maps/1"; + + await expect(service.calloutsUrl("de_mirage")).resolves.toBe( + "https://mirror.test/maps/1/de_mirage.callouts.json", + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("reads the key the latest manifest names, deduped into an older build", async () => { + await expect(service.calloutsUrl("de_mirage")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/25000000/de_mirage.callouts.json", + ); + await expect(service.calloutsUrl("de_nuke")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/25537370/de_nuke.callouts.json", + ); + expect(fetched("latest.json")).toBe(1); + expect(fetched("25537370/manifest.json")).toBe(1); + }); + + it("falls back to the pinned build for a map the manifest does not list", async () => { + await expect(service.calloutsUrl("de_unknown")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/24957633/de_unknown.callouts.json", + ); + }); + + it("falls back per asset when the map's entry has no callouts", async () => { + await expect(service.calloutsUrl("de_vertigo")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/24957633/de_vertigo.callouts.json", + ); + }); + + it("follows the manifest revision latest.json names", async () => { + files["latest.json"] = { + body: { ...pointer, manifest: "25537370/manifest.r2.json" }, + status: 200, + }; + files["25537370/manifest.r2.json"] = { + body: { + ...manifest, + maps: { de_anubis: { callouts: "25537370/de_anubis.callouts.json" } }, + }, + status: 200, + }; + + await expect(service.calloutsUrl("de_anubis")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/25537370/de_anubis.callouts.json", + ); + expect(fetched("25537370/manifest.json")).toBe(0); + }); + + it.each([ + ["latest.json", { ...pointer, version: 2 }], + ["25537370/manifest.json", { ...manifest, version: 2 }], + ])("treats an unknown %s version like an outage", async (key, body) => { + files[key] = { body, status: 200 }; + + await expect(service.calloutsUrl("de_nuke")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/24957633/de_nuke.callouts.json", + ); + }); + + it("falls back to the pinned build when latest.json is unreachable", async () => { + files["latest.json"] = { status: 502 }; + + await expect(service.calloutsUrl("de_mirage")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/24957633/de_mirage.callouts.json", + ); + await service.calloutsUrl("de_nuke"); + expect(fetched("latest.json")).toBe(1); + }); + + it("keeps the last manifest through a failed refresh", async () => { + const now = jest.spyOn(Date, "now").mockReturnValue(1_000_000); + await service.calloutsUrl("de_nuke"); + + files["latest.json"] = { status: 502 }; + now.mockReturnValue(1_000_000 + 11 * 60 * 1000); + + await expect(service.calloutsUrl("de_nuke")).resolves.toBe( + "https://demo-dl.5stack.gg/maps/25537370/de_nuke.callouts.json", + ); + expect(fetched("latest.json")).toBe(2); + }); + + it("syncs from the resolved URL", async () => { + files["25000000/de_mirage.callouts.json"] = { + body: { + callouts: [{ name: "Palace", boxes: [box([0, 0, 0], [10, 10, 10])] }], + }, + status: 200, + }; + const write = jest + .spyOn(service as any, "write") + .mockResolvedValue(undefined); + + await expect(service.sync("de_mirage_night")).resolves.toBe(1); + expect(write).toHaveBeenCalledWith( + "de_mirage", + [{ name: "Palace", boxes: [box([0, 0, 0], [10, 10, 10])] }], + "cdn", + ); + }); +}); diff --git a/src/utility/utility-callouts.service.ts b/src/utility/utility-callouts.service.ts index 84892f8e8..9d3a852fc 100644 --- a/src/utility/utility-callouts.service.ts +++ b/src/utility/utility-callouts.service.ts @@ -1,5 +1,10 @@ import { Injectable, Logger } from "@nestjs/common"; import { PostgresService } from "../postgres/postgres.service"; +import { + MapAssetsManifest, + MapAssetsPointer, + MapAssetsService, +} from "../map-assets/map-assets.service"; export type CalloutBox = { min: [number, number, number]; @@ -24,13 +29,13 @@ type CalloutRow = { @Injectable() export class UtilityCalloutsService { - // Callouts are published beside the collision meshes under one CS2 build, so - // this has to move with the browser's pin (web/nuxt.config.ts - // public.mapMeshCdn) and the demo parser's default -- otherwise the panel and - // the API can name the same throw differently after a map patch. Override - // with MAP_MESH_CDN. - private static readonly DEFAULT_CDN = - "https://demo-dl.5stack.gg/maps/24957633"; + // Callouts resolve the same way the panel and the demo parser resolve meshes + // (latest.json -> that build's manifest), otherwise the two can name the same + // throw differently after a map patch. MAP_MESH_CDN pins a flat + // /.callouts.json directory instead. + private static readonly LATEST_TTL_MS = 10 * 60 * 1000; + + private static readonly LATEST_RETRY_MS = 30 * 1000; // How far outside every place volume a point may sit and still be named. The // volumes do not tile a map, and a grenade rests on top of geometry as often @@ -72,6 +77,13 @@ export class UtilityCalloutsService { { rows: CalloutRow[]; expires: number } >(); + private latest: { manifest: MapAssetsManifest | null; expires: number } = { + manifest: null, + expires: 0, + }; + + private readonly manifests = new Map(); + constructor( private readonly logger: Logger, private readonly postgres: PostgresService, @@ -120,8 +132,8 @@ export class UtilityCalloutsService { /** * Pull the published extract for one map. Best effort on purpose: the CDN is - * not on the critical path of anything, and a jsDelivr blip must leave the - * rows already in the table alone rather than emptying the table. + * not on the critical path of anything, and a CDN blip must leave the rows + * already in the table alone rather than emptying the table. */ public async sync(mapName: string): Promise { const map = UtilityCalloutsService.normalizeMapName(mapName); @@ -129,11 +141,13 @@ export class UtilityCalloutsService { return 0; } - const base = process.env.MAP_MESH_CDN || UtilityCalloutsService.DEFAULT_CDN; - let callouts: MapCallout[]; try { - const response = await fetch(`${base}/${map}.callouts.json`); + const url = await this.calloutsUrl(map); + if (!url) { + return 0; + } + const response = await fetch(url); if (!response.ok) { return 0; } @@ -154,6 +168,69 @@ export class UtilityCalloutsService { return callouts.length; } + public async calloutsUrl(map: string): Promise { + const base = process.env.MAP_MESH_CDN; + if (base) { + return `${base}/${map}.callouts.json`; + } + + return MapAssetsService.assetUrl( + await this.latestManifest(), + map, + "callouts", + ); + } + + // A failed refresh keeps serving the last manifest that loaded; only a + // process that has never read latest.json falls back to the pinned build. + // latest.json may name a manifest revision (/manifest.r2.json), so its + // key is followed as given. + private async latestManifest(): Promise { + if (this.latest.expires > Date.now()) { + return this.latest.manifest; + } + + try { + const pointer = + await MapAssetsService.fetchIndex("latest.json"); + if (pointer && pointer.version !== MapAssetsService.INDEX_VERSION) { + throw new Error(`latest.json is version ${pointer.version}`); + } + if (!MapAssetsService.isSafeKey(pointer?.manifest)) { + throw new Error("latest.json names no manifest"); + } + + let manifest = this.manifests.get(pointer.manifest); + if (!manifest) { + manifest = await MapAssetsService.fetchIndex( + pointer.manifest, + ); + if (!manifest?.maps) { + throw new Error(`${pointer.manifest} is not published`); + } + if (manifest.version !== MapAssetsService.INDEX_VERSION) { + throw new Error(`${pointer.manifest} is version ${manifest.version}`); + } + this.manifests.set(pointer.manifest, manifest); + } + + this.latest = { + manifest, + expires: Date.now() + UtilityCalloutsService.LATEST_TTL_MS, + }; + } catch (error) { + this.logger.warn( + `unable to resolve the latest map assets: ${(error as Error)?.message}`, + ); + this.latest = { + manifest: this.latest.manifest, + expires: Date.now() + UtilityCalloutsService.LATEST_RETRY_MS, + }; + } + + return this.latest.manifest; + } + public async syncAll(): Promise<{ maps: number; callouts: number }> { const maps = await this.postgres.query>( `SELECT DISTINCT name diff --git a/src/utility/utility-drift.service.ts b/src/utility/utility-drift.service.ts index 614af2979..2cb6e51e0 100644 --- a/src/utility/utility-drift.service.ts +++ b/src/utility/utility-drift.service.ts @@ -160,9 +160,19 @@ export class UtilityDriftService { await this.record(scan.id, batch, answer.data.results ?? []); scanned += batch.length; + // The caveats are the parser's warning label for every number in the + // scan (and name a revision pair without grenade clips), so they travel + // with the scan rather than being dropped at the boundary. await this.postgres.query( - `UPDATE public.utility_drift_scans SET scanned = $2::int WHERE id = $1::uuid`, - [scan.id, scanned], + `UPDATE public.utility_drift_scans + SET scanned = $2::int, + caveats = COALESCE($3::jsonb, caveats) + WHERE id = $1::uuid`, + [ + scan.id, + scanned, + answer.data.caveats ? JSON.stringify(answer.data.caveats) : null, + ], ); } diff --git a/test/map-asset-builds.spec.ts b/test/map-asset-builds.spec.ts new file mode 100644 index 000000000..d343cc22b --- /dev/null +++ b/test/map-asset-builds.spec.ts @@ -0,0 +1,281 @@ +import { Logger } from "@nestjs/common"; +import { PostgresService } from "./../src/postgres/postgres.service"; +import { MapAssetsService } from "./../src/map-assets/map-assets.service"; +import { bootMigratedDb, SqlTestDb } from "./utils/sql-test-db"; + +// One row per CS2 build, claimed by whichever node reports the build first; +// every other node reporting it must not queue a second build. +describe("map asset builds (SQL-driven)", () => { + const originalDomain = process.env.WEB_DOMAIN; + const originalFetch = global.fetch; + + let db: SqlTestDb; + let postgres: PostgresService; + let queue: { add: jest.Mock; getJob: jest.Mock }; + let loggingService: { getJobStatus: jest.Mock; getJobPod: jest.Mock }; + + beforeAll(async () => { + db = await bootMigratedDb("MapAssetBuildsTest"); + postgres = db.postgres; + }, 600_000); + + afterAll(async () => { + await db?.stop(); + }); + + beforeEach(async () => { + process.env.WEB_DOMAIN = "5stack.gg"; + await postgres.query("DELETE FROM map_asset_builds"); + await postgres.query("DELETE FROM game_server_nodes"); + await postgres.query("DELETE FROM game_versions"); + await postgres.query( + "DELETE FROM settings WHERE name = 'map_assets_auto_build'", + ); + await postgres.query( + `INSERT INTO server_regions (value, description) + VALUES ('TestRegion', 'TestRegion') ON CONFLICT (value) DO NOTHING`, + ); + await postgres.query( + `INSERT INTO game_versions (build_id, version, description, current, updated_at) + VALUES (25537370, '1.0', 'current', true, now()), + (24957633, '0.9', 'previous', null, now())`, + ); + await postgres.query( + `INSERT INTO game_server_nodes (id, status, enabled, region, build_id) + VALUES ('node-a', 'Online', true, 'TestRegion', 25537370)`, + ); + queue = { + add: jest.fn().mockResolvedValue({}), + getJob: jest.fn().mockResolvedValue(undefined), + }; + loggingService = { + getJobStatus: jest.fn(), + getJobPod: jest.fn().mockResolvedValue(undefined), + }; + }); + + afterEach(() => { + process.env.WEB_DOMAIN = originalDomain; + global.fetch = originalFetch; + }); + + const enableAutoBuild = async () => { + await postgres.query( + `INSERT INTO settings (name, value) VALUES ('map_assets_auto_build', 'true')`, + ); + }; + + const service = () => { + const mapAssets = new MapAssetsService( + new Logger("MapAssetBuildsTest"), + { get: () => ({ namespace: "5stack" }) } as never, + postgres, + loggingService as never, + queue as never, + ); + (mapAssets as any).batchApi = { + deleteNamespacedJob: jest.fn().mockResolvedValue({}), + createNamespacedJob: jest.fn().mockResolvedValue({}), + }; + (mapAssets as any).coreApi = { + readNamespacedPodLog: jest.fn().mockResolvedValue("de_anubis failed\n"), + }; + return mapAssets; + }; + + const serve = (files: Record) => { + global.fetch = jest.fn(async (url: string) => { + const key = url.replace("https://demo-dl.5stack.gg/maps/", ""); + if (!(key in files)) { + return { + ok: false, + status: 404, + json: async (): Promise => null, + }; + } + return { ok: true, status: 200, json: async () => files[key] }; + }) as never; + }; + + const row = async (buildId: string) => { + const [found] = await postgres.query< + Array<{ + status: string; + manifest: string | null; + maps: unknown; + failed: Array | null; + failed_view: Array | null; + error: string | null; + started_at: Date | null; + finished_at: Date | null; + created_at: Date; + updated_at: Date; + }> + >( + `SELECT status, manifest, maps, failed, failed_view, error, + started_at, finished_at, created_at, updated_at + FROM map_asset_builds + WHERE build_id = $1`, + [buildId], + ); + return found; + }; + + it("does not build automatically until an operator turns it on", async () => { + await expect(service().queueBuild("node-a", 25537370)).resolves.toBe(false); + expect(await row("25537370")).toBeUndefined(); + + await expect(service().queueManualBuild("node-a")).resolves.toBe(true); + expect((await row("25537370")).status).toBe("Pending"); + }); + + it("claims the current build once, however many nodes report it", async () => { + await enableAutoBuild(); + const mapAssets = service(); + + await expect(mapAssets.queueBuild("node-a", 25537370)).resolves.toBe(true); + await expect(mapAssets.queueBuild("node-b", 25537370)).resolves.toBe(false); + + expect(queue.add).toHaveBeenCalledTimes(1); + expect((await row("25537370")).status).toBe("Pending"); + }); + + it("ignores a build that is not the current one", async () => { + await enableAutoBuild(); + + await expect(service().queueBuild("node-a", 24957633)).resolves.toBe(false); + expect(await row("24957633")).toBeUndefined(); + }); + + it("only knows the five lifecycle states", async () => { + await expect( + postgres.query( + `INSERT INTO map_asset_builds (build_id, status) VALUES ('1', 'Done')`, + ), + ).rejects.toThrow(/map_asset_builds_status_check/); + await postgres.query( + `INSERT INTO map_asset_builds (build_id, status) VALUES ('2', 'Partial')`, + ); + }); + + it.each(["Failed", "Partial"])( + "retries a %s build by hand", + async (status) => { + await postgres.query( + `INSERT INTO map_asset_builds (build_id, status, error) + VALUES ('25537370', $1, 'boom')`, + [status], + ); + + await expect(service().queueManualBuild("node-a")).resolves.toBe(true); + expect(await row("25537370")).toMatchObject({ + status: "Pending", + error: null, + }); + }, + ); + + it("leaves a published build, or one already running, alone", async () => { + await postgres.query( + `INSERT INTO map_asset_builds (build_id, status) VALUES ('25537370', 'Published')`, + ); + await expect(service().queueManualBuild("node-a")).resolves.toBe(false); + + await postgres.query( + `UPDATE map_asset_builds SET status = 'Building' WHERE build_id = '25537370'`, + ); + queue.getJob.mockResolvedValueOnce({ id: "map-assets.25537370" }); + await expect(service().queueManualBuild("node-a")).resolves.toBe(false); + expect((await row("25537370")).status).toBe("Building"); + expect(queue.add).not.toHaveBeenCalled(); + }); + + it("records a published build and the manifest revision it came from", async () => { + const maps = { de_mirage: { tri: "25000000/de_mirage.tri.gz" } }; + await postgres.query( + `INSERT INTO map_asset_builds (build_id, created_at, updated_at) + VALUES ('25537370', now() - interval '1 day', now() - interval '1 day')`, + ); + loggingService.getJobStatus + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce({ succeeded: 1 }); + serve({ + "latest.json": { + version: 1, + build: "25537370", + manifest: "25537370/manifest.r2.json", + }, + "25537370/manifest.r2.json": { version: 1, build: "25537370", maps }, + }); + + await service().build("node-a", "25537370"); + + const published = await row("25537370"); + expect(published).toMatchObject({ + status: "Published", + manifest: "25537370/manifest.r2.json", + maps, + failed: [], + failed_view: [], + error: null, + }); + expect(published.started_at).toBeInstanceOf(Date); + expect(published.finished_at).toBeInstanceOf(Date); + expect(published.updated_at.getTime()).toBeGreaterThan( + published.created_at.getTime() + 60 * 60 * 1000, + ); + }); + + it("records a partial publish, and keeps it when the retry fails outright", async () => { + const maps = { de_mirage: { tri: "25537370/de_mirage.tri.gz" } }; + loggingService.getJobStatus + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce({ failed: 1 }); + loggingService.getJobPod.mockResolvedValue({ + metadata: { name: "map-assets-25537370-abcde" }, + status: { + containerStatuses: [ + { + name: "map-assets", + state: { terminated: { exitCode: 2, reason: "Error" } }, + }, + ], + }, + }); + serve({ + "latest.json": { + version: 1, + build: "25537370", + manifest: "25537370/manifest.json", + }, + "25537370/manifest.json": { + version: 1, + build: "25537370", + maps, + failed: ["de_anubis"], + failed_view: [], + }, + }); + + await service().build("node-a", "25537370"); + + expect(await row("25537370")).toMatchObject({ + status: "Partial", + manifest: "25537370/manifest.json", + maps, + failed: ["de_anubis"], + failed_view: [], + error: "de_anubis failed", + }); + + loggingService.getJobStatus + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce(undefined); + await service().build("node-a", "25537370"); + + const retried = await row("25537370"); + expect(retried.status).toBe("Failed"); + expect(retried.maps).toEqual(maps); + expect(retried.failed).toEqual(["de_anubis"]); + }); +}); diff --git a/test/utility-drift.spec.ts b/test/utility-drift.spec.ts index dccaa8da9..88d4e9ce4 100644 --- a/test/utility-drift.spec.ts +++ b/test/utility-drift.spec.ts @@ -49,6 +49,7 @@ describe("utility drift scans (SQL-driven)", () => { verdicts: ( request: ParsedDriftRequest, ) => Array | null = () => [], + caveats?: Array, ) { return { drift: jest.fn(async (request: ParsedDriftRequest) => { @@ -64,7 +65,7 @@ describe("utility drift scans (SQL-driven)", () => { } return { - data: { results }, + data: { results, caveats }, status: 200, error: null as string | null, }; @@ -318,6 +319,35 @@ describe("utility drift scans (SQL-driven)", () => { expect(types).not.toContain("HighExplosive"); }); + // The caveats are the only thing standing between a screen of distances + // and someone reading them as landing spots, so they outlive the request. + it("keeps the parser's caveats with the scan", async () => { + const author = await fx.player(); + await insertLineup(author); + + const caveats = [ + "comparison points are simulator output, not real landings", + "only one revision publishes grenade clips (from: 0 triangles, to: 950)", + ]; + const drift = service( + parserStub( + () => [{ index: 0, verdict: "unchanged" as const }], + caveats, + ), + ); + + const scan = await drift.startScan(admin(author), { + map_name: "de_mirage", + }); + await drift.runScan(scan.scan_id); + + const [row] = await postgres.query }>>( + `SELECT caveats FROM utility_drift_scans WHERE id = $1::uuid`, + [scan.scan_id], + ); + expect(row.caveats).toEqual(caveats); + }); + it("carries the mesh revisions the scan was started with", async () => { const author = await fx.player(); await insertLineup(author);