feature: map assets builds

hasura/metadata/actions.graphql

@@ -84,6 +84,12 @@ type Mutation {
   ): SuccessOutput
 }
 
+type Mutation {
+  buildMapAssets(
+    game_server_node_id: uuid!
+  ): SuccessOutput
+}
+
 type Mutation {
   callForOrganizer(
     match_id: String!

hasura/metadata/actions.yaml

@@ -165,6 +165,14 @@ actions:
     permissions:
       - role: administrator
     comment: Launch a Vulkan shader pre-bake Job on a GPU node
+  - name: buildMapAssets
+    definition:
+      kind: synchronous
+      handler: '{{HASURA_GRAPHQL_ACTIONS_HOOK}}'
+      forward_client_headers: true
+    permissions:
+      - role: administrator
+    comment: Build and publish map assets from a node's CS2 install (5stack.gg only)
   - name: callForOrganizer
     definition:
       kind: synchronous

hasura/metadata/databases/default/tables/public_map_asset_builds.yaml

@@ -0,0 +1,21 @@
+table:
+  name: map_asset_builds
+  schema: public
+select_permissions:
+  - role: administrator
+    permission:
+      columns:
+        - build_id
+        - status
+        - started_at
+        - finished_at
+        - manifest
+        - maps
+        - failed
+        - failed_view
+        - error
+        - created_at
+        - updated_at
+      filter: {}
+      allow_aggregations: true
+    comment: ""

hasura/metadata/databases/default/tables/public_utility_drift_scans.yaml

@@ -31,6 +31,7 @@ select_permissions:
         - broken
         - unsimulatable
         - max_distance
+        - caveats
         - requested_by_steam_id
         - started_at
         - finished_at

hasura/metadata/databases/default/tables/tables.yaml

@@ -102,6 +102,7 @@
 - "!include public_league_teams.yaml"
 - "!include public_lobbies.yaml"
 - "!include public_lobby_players.yaml"
+- "!include public_map_asset_builds.yaml"
 - "!include public_map_callouts.yaml"
 - "!include public_map_pools.yaml"
 - "!include public_maps.yaml"

hasura/migrations/default/1887000000000_map_asset_builds/down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS "public"."map_asset_builds";

hasura/migrations/default/1887000000000_map_asset_builds/up.sql

@@ -0,0 +1,16 @@
+CREATE TABLE IF NOT EXISTS "public"."map_asset_builds" (
+    "build_id" text NOT NULL,
+    "status" text NOT NULL DEFAULT 'Pending',
+    "started_at" timestamptz,
+    "finished_at" timestamptz,
+    "manifest" text,
+    "maps" jsonb,
+    "failed" jsonb,
+    "failed_view" jsonb,
+    "error" text,
+    "created_at" timestamptz NOT NULL DEFAULT now(),
+    "updated_at" timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY ("build_id"),
+    CONSTRAINT "map_asset_builds_status_check"
+        CHECK ("status" IN ('Pending', 'Building', 'Published', 'Partial', 'Failed'))
+);

hasura/migrations/default/1887000000100_utility_drift_scan_caveats/down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "public"."utility_drift_scans"
+    DROP COLUMN IF EXISTS "caveats";

hasura/migrations/default/1887000000100_utility_drift_scan_caveats/up.sql

@@ -0,0 +1,2 @@
+ALTER TABLE "public"."utility_drift_scans"
+    ADD COLUMN IF NOT EXISTS "caveats" jsonb;

hasura/triggers/map_asset_builds.sql

@@ -0,0 +1,5 @@
+DROP TRIGGER IF EXISTS "set_public_map_asset_builds_updated_at" ON "public"."map_asset_builds";
+CREATE TRIGGER "set_public_map_asset_builds_updated_at"
+BEFORE UPDATE ON "public"."map_asset_builds"
+FOR EACH ROW
+EXECUTE PROCEDURE "public"."set_current_timestamp_updated_at"();

src/game-server-node/game-server-node.module.ts

@@ -33,6 +33,7 @@ import { PluginRuntimeModule } from "src/plugin-runtime/plugin-runtime.module";
 import { GamePluginsModule } from "src/game-plugins/game-plugins.module";
 import { BakeShaders } from "./jobs/BakeShaders";
 import { ValidateGamedata } from "./jobs/ValidateGamedata";
+import { MapAssetsModule } from "src/map-assets/map-assets.module";
 
 @Module({
   providers: [
@@ -59,6 +60,7 @@ import { ValidateGamedata } from "./jobs/ValidateGamedata";
     GameStreamerModule,
     PluginRuntimeModule,
     GamePluginsModule,
+    MapAssetsModule,
     BullModule.registerQueue(
       {
         name: GameServerQueues.GameUpdate,

src/game-server-node/game-server-node.service.ts

@@ -24,6 +24,7 @@ import { GameServerQueues } from "./enums/GameServerQueues";
 import { NotificationsService } from "src/notifications/notifications.service";
 import { PluginRuntimeService } from "src/plugin-runtime/plugin-runtime.service";
 import { PluginRuntime } from "src/configs/types/GameServersConfig";
+import { MapAssetsService } from "src/map-assets/map-assets.service";
 
 export type GamedataValidationRuntime = PluginRuntime;
 
@@ -73,6 +74,7 @@ export class GameServerNodeService {
     protected readonly loggingService: LoggingService,
     protected readonly notifications: NotificationsService,
     protected readonly pluginRuntimeService: PluginRuntimeService,
+    protected readonly mapAssets: MapAssetsService,
     @InjectQueue(GameServerQueues.ValidateGamedata)
     private readonly validateGamedataQueue: Queue,
   ) {
@@ -347,7 +349,18 @@ export class GameServerNodeService {
       csBulid &&
       game_server_nodes_by_pk.build_id !== csBulid
     ) {
-      await this.queueGamedataValidation(node, csBulid);
+      // Map assets and the gamedata validator both read this node's install,
+      // so when a validation is queued the map-assets build is chained after it
+      // (see ValidateGamedata) rather than run beside it on the same node.
+      const validating = await this.queueGamedataValidation(node, csBulid);
+      if (!validating) {
+        await this.mapAssets.queueBuild(node, csBulid).catch((error) => {
+          this.logger.warn(
+            `[map-assets] unable to queue build ${csBulid}`,
+            error,
+          );
+        });
+      }
     }
 
     if (transitionedFromOffline && game_server_nodes_by_pk.build_id) {
@@ -942,14 +955,14 @@ export class GameServerNodeService {
   private async queueGamedataValidation(
     gameServerNodeId: string,
     buildId: number,
-  ) {
+  ): Promise<boolean> {
     if (process.env.WEB_DOMAIN !== "5stack.gg") {
-      return;
+      return false;
     }
 
     const currentBuild = await this.getCurrentBuild();
     if (buildId !== currentBuild) {
-      return;
+      return false;
     }
 
     const { gamedata_signature_validations } = await this.hasura.query({
@@ -966,14 +979,15 @@ export class GameServerNodeService {
     });
 
     if (gamedata_signature_validations.length > 0) {
-      return;
+      return false;
     }
 
     await this.validateGamedataQueue.add(
       "ValidateGamedata",
       {
         gameServerNodeId,
         buildId,
+        buildMapAssets: true,
       },
       {
         jobId: `validate.${buildId}.auto`,
@@ -982,6 +996,8 @@ export class GameServerNodeService {
         removeOnFail: true,
       },
     );
+
+    return true;
   }
 
   public async validateGamedata(

src/game-server-node/jobs/ValidateGamedata.spec.ts

@@ -0,0 +1,59 @@
+import { ValidateGamedata } from "./ValidateGamedata";
+
+describe("ValidateGamedata", () => {
+  let gameServerNodeService: { validateGamedata: jest.Mock };
+  let mapAssets: { queueBuild: jest.Mock };
+  let job: ValidateGamedata;
+
+  beforeEach(() => {
+    gameServerNodeService = {
+      validateGamedata: jest
+        .fn()
+        .mockResolvedValue({ status: "pass", broken: [] }),
+    };
+    mapAssets = { queueBuild: jest.fn().mockResolvedValue(true) };
+    job = new ValidateGamedata(
+      { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any,
+      { send: jest.fn() } as any,
+      gameServerNodeService as any,
+      mapAssets as any,
+    );
+  });
+
+  const run = (data: Record<string, unknown>) =>
+    job.process({
+      data: { gameServerNodeId: "node-1", buildId: 25537370, ...data },
+    } as any);
+
+  it("starts the chained map-assets build once validation finishes", async () => {
+    const order: Array<string> = [];
+    gameServerNodeService.validateGamedata.mockImplementation(async () => {
+      order.push("validate");
+      return { status: "pass", broken: [] };
+    });
+    mapAssets.queueBuild.mockImplementation(async () => {
+      order.push("map-assets");
+      return true;
+    });
+
+    await run({ buildMapAssets: true });
+
+    expect(order).toEqual(["validate", "map-assets"]);
+    expect(mapAssets.queueBuild).toHaveBeenCalledWith("node-1", 25537370);
+  });
+
+  it("still starts it when validation fails outright", async () => {
+    gameServerNodeService.validateGamedata.mockRejectedValue(
+      new Error("k8s down"),
+    );
+
+    await expect(run({ buildMapAssets: true })).rejects.toThrow("k8s down");
+    expect(mapAssets.queueBuild).toHaveBeenCalledWith("node-1", 25537370);
+  });
+
+  it("leaves map assets alone for a validation nobody chained to", async () => {
+    await run({});
+
+    expect(mapAssets.queueBuild).not.toHaveBeenCalled();
+  });
+});

src/game-server-node/jobs/ValidateGamedata.ts

@@ -9,11 +9,13 @@ import {
   GameServerNodeService,
   GamedataValidationEntry,
 } from "../game-server-node.service";
+import { MapAssetsService } from "src/map-assets/map-assets.service";
 
 type ValidateGamedataData = {
   gameServerNodeId: string;
   buildId: number;
   branch?: string;
+  buildMapAssets?: boolean;
 };
 
 const GAMEDATA_ROUTING = {
@@ -36,11 +38,31 @@ export class ValidateGamedata extends WorkerHost {
     protected readonly logger: Logger,
     protected readonly notifications: NotificationsService,
     protected readonly gameServerNodeService: GameServerNodeService,
+    protected readonly mapAssets: MapAssetsService,
   ) {
     super();
   }
 
+  // The map-assets build for a new CS2 version waits for this validation,
+  // pass or fail, so the two never share the node's install at once.
   async process(job: Job<ValidateGamedataData>): Promise<void> {
+    try {
+      await this.validate(job);
+    } finally {
+      if (job.data.buildMapAssets) {
+        await this.mapAssets
+          .queueBuild(job.data.gameServerNodeId, job.data.buildId)
+          .catch((error) => {
+            this.logger.warn(
+              `[map-assets] unable to queue build ${job.data.buildId}`,
+              error,
+            );
+          });
+      }
+    }
+  }
+
+  private async validate(job: Job<ValidateGamedataData>): Promise<void> {
     const { gameServerNodeId, buildId } = job.data;
     const branch = job.data.branch ?? "public";
 

src/map-assets/enums/MapAssetsQueues.ts

@@ -0,0 +1,3 @@
+export enum MapAssetsQueues {
+  BuildMapAssets = "build-map-assets",
+}

src/map-assets/jobs/BuildMapAssets.ts

@@ -0,0 +1,39 @@
+import { WorkerHost } from "@nestjs/bullmq";
+import { Job } from "bullmq";
+import { Logger } from "@nestjs/common";
+import { UseQueue } from "../../utilities/QueueProcessors";
+import { MapAssetsQueues } from "../enums/MapAssetsQueues";
+import { MapAssetsService } from "../map-assets.service";
+
+type BuildMapAssetsData = {
+  gameServerNodeId: string;
+  buildId: string;
+};
+
+@UseQueue("MapAssets", MapAssetsQueues.BuildMapAssets)
+export class BuildMapAssets extends WorkerHost {
+  constructor(
+    protected readonly logger: Logger,
+    protected readonly mapAssets: MapAssetsService,
+  ) {
+    super();
+  }
+
+  async process(job: Job<BuildMapAssetsData>): Promise<void> {
+    const { gameServerNodeId } = job.data;
+    const buildId = String(job.data.buildId);
+
+    const outcome = await this.mapAssets.build(gameServerNodeId, buildId);
+
+    if (outcome.status === "Failed") {
+      this.logger.error(
+        `[map-assets] build ${buildId} failed on ${gameServerNodeId}: ${outcome.error}`,
+      );
+      return;
+    }
+
+    this.logger.log(
+      `[map-assets] published build ${buildId} (${Object.keys(outcome.maps ?? {}).length} maps)`,
+    );
+  }
+}

src/map-assets/map-assets.controller.ts

@@ -0,0 +1,15 @@
+import { Controller } from "@nestjs/common";
+import { HasuraAction } from "../hasura/hasura.controller";
+import { MapAssetsService } from "./map-assets.service";
+
+@Controller("map-assets")
+export class MapAssetsController {
+  constructor(private readonly mapAssets: MapAssetsService) {}
+
+  @HasuraAction()
+  public async buildMapAssets(data: { game_server_node_id: string }) {
+    return {
+      success: await this.mapAssets.queueManualBuild(data.game_server_node_id),
+    };
+  }
+}

src/map-assets/map-assets.module.ts

@@ -0,0 +1,35 @@
+import { Module } from "@nestjs/common";
+import { BullModule } from "@nestjs/bullmq";
+import { BullBoardModule } from "@bull-board/nestjs";
+import { BullMQAdapter } from "@bull-board/api/bullMQAdapter";
+import { PostgresModule } from "../postgres/postgres.module";
+import { K8sModule } from "../k8s/k8s.module";
+import { getQueuesProcessors } from "../utilities/QueueProcessors";
+import { loggerFactory } from "../utilities/LoggerFactory";
+import { MapAssetsQueues } from "./enums/MapAssetsQueues";
+import { MapAssetsService } from "./map-assets.service";
+import { MapAssetsController } from "./map-assets.controller";
+import { BuildMapAssets } from "./jobs/BuildMapAssets";
+
+@Module({
+  imports: [
+    PostgresModule,
+    K8sModule,
+    BullModule.registerQueue({
+      name: MapAssetsQueues.BuildMapAssets,
+    }),
+    BullBoardModule.forFeature({
+      name: MapAssetsQueues.BuildMapAssets,
+      adapter: BullMQAdapter,
+    }),
+  ],
+  controllers: [MapAssetsController],
+  providers: [
+    MapAssetsService,
+    BuildMapAssets,
+    ...getQueuesProcessors("MapAssets"),
+    loggerFactory(),
+  ],
+  exports: [MapAssetsService],
+})
+export class MapAssetsModule {}