bug: only send server and node status notifications that make sense

hasura/metadata/databases/default/tables/public_game_server_nodes.yaml

@@ -61,6 +61,7 @@ select_permissions:
   - role: administrator
     permission:
       columns:
+        - accepting_new_matches
         - build_id
         - cpu_cores_per_socket
         - cpu_frequency_info

hasura/migrations/default/1889000000700_game_server_node_accepting_new_matches/down.sql

@@ -0,0 +1,2 @@
+ALTER TABLE public.game_server_nodes
+    DROP COLUMN IF EXISTS accepting_new_matches;

hasura/migrations/default/1889000000700_game_server_node_accepting_new_matches/up.sql

@@ -0,0 +1,6 @@
+ALTER TABLE public.game_server_nodes
+    ADD COLUMN IF NOT EXISTS accepting_new_matches boolean NOT NULL DEFAULT true;
+
+UPDATE public.game_server_nodes
+   SET accepting_new_matches = false
+ WHERE status = 'NotAcceptingNewMatches';

src/dedicated-servers/dedicated-servers.module.ts

@@ -18,6 +18,7 @@ import { SystemModule } from "src/system/system.module";
 import { PluginRuntimeModule } from "src/plugin-runtime/plugin-runtime.module";
 import { GamePluginsModule } from "../game-plugins/game-plugins.module";
 import { PostgresModule } from "../postgres/postgres.module";
+import { NotificationsModule } from "../notifications/notifications.module";
 
 @Module({
   imports: [
@@ -36,6 +37,7 @@ import { PostgresModule } from "../postgres/postgres.module";
     PluginRuntimeModule,
     GamePluginsModule,
     PostgresModule,
+    NotificationsModule,
   ],
   providers: [
     DedicatedServersService,

src/dedicated-servers/dedicated-servers.service.spec.ts

@@ -4,13 +4,15 @@ import { DedicatedServersService } from "./dedicated-servers.service";
 // to interleave: both removed, one created, and the other's AlreadyExists
 // handler deleted the deployment the first had just created.
 describe("DedicatedServersService.rebuildDedicatedServer", () => {
+  const redis = { set: jest.fn() };
   const service = new DedicatedServersService(
     { log: jest.fn(), error: jest.fn(), verbose: jest.fn() } as never,
     { get: () => ({ namespace: "5stack" }) } as never,
     null as never,
     null as never,
     null as never,
-    { getConnection: () => ({}) } as never,
+    { getConnection: () => redis } as never,
+    null as never,
     null as never,
     null as never,
     null as never,
@@ -22,6 +24,7 @@ describe("DedicatedServersService.rebuildDedicatedServer", () => {
 
   beforeEach(() => {
     steps.length = 0;
+    redis.set.mockClear();
 
     jest
       .spyOn(service, "removeDedicatedServer")
@@ -70,6 +73,23 @@ describe("DedicatedServersService.rebuildDedicatedServer", () => {
     expect(steps).toEqual(["remove a"]);
   });
 
+  it("gives a server that will start again time to boot", async () => {
+    await service.rebuildDedicatedServer("a");
+
+    expect(redis.set).toHaveBeenCalledWith(
+      "dedicated-servers:restarting:a",
+      "1",
+      "PX",
+      5 * 60 * 1000,
+    );
+  });
+
+  it("gives no boot time to a server being taken down", async () => {
+    await service.rebuildDedicatedServer("a", false);
+
+    expect(redis.set).not.toHaveBeenCalled();
+  });
+
   it("keeps going after a rebuild that failed", async () => {
     jest
       .spyOn(service, "setupDedicatedServer")
@@ -85,6 +105,278 @@ describe("DedicatedServersService.rebuildDedicatedServer", () => {
   });
 });
 
+class FakeRedis {
+  private values = new Map<string, { value: string; expiresAt?: number }>();
+  private hashes = new Map<string, Map<string, string>>();
+  public now = Date.now();
+
+  async set(key: string, value: string, mode?: string, ms?: number) {
+    this.values.set(key, {
+      value,
+      expiresAt: mode === "PX" ? this.now + ms : undefined,
+    });
+    return "OK";
+  }
+
+  async pttl(key: string) {
+    const entry = this.values.get(key);
+    if (!entry || (entry.expiresAt && entry.expiresAt <= this.now)) {
+      return -2;
+    }
+    return entry.expiresAt ? entry.expiresAt - this.now : -1;
+  }
+
+  private hash(key: string) {
+    if (!this.hashes.has(key)) {
+      this.hashes.set(key, new Map());
+    }
+    return this.hashes.get(key);
+  }
+
+  async hsetnx(key: string, field: string, value: string) {
+    if (this.hash(key).has(field)) {
+      return 0;
+    }
+    this.hash(key).set(field, value);
+    return 1;
+  }
+
+  async hset(key: string, field: string, value: string) {
+    this.hash(key).set(field, value);
+    return 1;
+  }
+
+  async hget(key: string, field: string) {
+    return this.hash(key).get(field) ?? null;
+  }
+
+  async hdel(key: string, field: string) {
+    return this.hash(key).delete(field) ? 1 : 0;
+  }
+
+  async expire() {
+    return 1;
+  }
+}
+
+describe("DedicatedServersService.pingDedicatedServer", () => {
+  let redis: FakeRedis;
+  let row: Record<string, unknown>;
+  let reachable: boolean;
+  let hasura: { query: jest.Mock; mutation: jest.Mock };
+  let notifications: { send: jest.Mock };
+  let service: DedicatedServersService;
+
+  beforeEach(() => {
+    redis = new FakeRedis();
+    reachable = true;
+    row = {
+      game: "cs2",
+      label: "Retakes #1",
+      enabled: true,
+      connected: true,
+      steam_relay: null,
+      game_server_node_id: "node-1",
+      server_region: { steam_relay: false },
+    };
+    hasura = {
+      query: jest.fn(async () => ({ servers_by_pk: { ...row } })),
+      mutation: jest.fn(async (mutation: Record<string, any>) => {
+        Object.assign(row, mutation.update_servers_by_pk?.__args._set ?? {});
+        return {};
+      }),
+    };
+    notifications = { send: jest.fn().mockResolvedValue(undefined) };
+    const rcon = {
+      connect: jest.fn(async () =>
+        reachable
+          ? {
+              send: async () =>
+                JSON.stringify({
+                  server: {
+                    steamid: null,
+                    clients_human: 3,
+                    map: "de_inferno",
+                  },
+                }),
+            }
+          : null,
+      ),
+      disconnect: jest.fn(),
+    };
+
+    service = new DedicatedServersService(
+      { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as never,
+      { get: () => ({ namespace: "5stack" }) } as never,
+      hasura as never,
+      null as never,
+      rcon as never,
+      { getConnection: () => redis } as never,
+      { restartDeployment: jest.fn() } as never,
+      null as never,
+      null as never,
+      null as never,
+      notifications as never,
+    );
+  });
+
+  const minutes = (count: number) => {
+    redis.now += count * 60 * 1000;
+    jest.setSystemTime(redis.now);
+  };
+
+  beforeAll(() => {
+    jest.useFakeTimers({ doNotFake: ["setTimeout", "setImmediate"] });
+  });
+
+  afterAll(() => {
+    jest.useRealTimers();
+  });
+
+  beforeEach(() => {
+    jest.setSystemTime(redis.now);
+  });
+
+  const alerts = () => notifications.send.mock.calls.length;
+
+  it("does not mark an unreachable server connected", async () => {
+    row.connected = false;
+    reachable = false;
+
+    await service.pingDedicatedServer("server-1");
+
+    expect(row.connected).toBe(false);
+  });
+
+  it("marks a reachable server connected", async () => {
+    row.connected = false;
+
+    await service.pingDedicatedServer("server-1");
+
+    expect(row.connected).toBe(true);
+  });
+
+  it("stays quiet about a server that just stopped answering", async () => {
+    reachable = false;
+
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+
+    expect(alerts()).toBe(0);
+    expect(row.connected).toBe(true);
+  });
+
+  it("reports a server unreachable for two minutes, once", async () => {
+    reachable = false;
+
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+
+    expect(notifications.send).toHaveBeenCalledTimes(1);
+    expect(notifications.send).toHaveBeenCalledWith(
+      "DedicatedServerRconStatus",
+      expect.objectContaining({ title: "Dedicated Server RCON Error" }),
+      undefined,
+      expect.any(Number),
+    );
+    expect(row.connected).toBe(false);
+  });
+
+  it("starts over once the server answers again", async () => {
+    reachable = false;
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    reachable = true;
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    reachable = false;
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+
+    expect(alerts()).toBe(0);
+  });
+
+  it("waits out a restart before reporting", async () => {
+    await service.restartDedicatedServer("server-1");
+    reachable = false;
+
+    for (let minute = 0; minute < 5; minute++) {
+      await service.pingDedicatedServer("server-1");
+      minutes(1);
+    }
+    expect(alerts()).toBe(0);
+
+    await service.pingDedicatedServer("server-1");
+    expect(alerts()).toBe(1);
+  });
+
+  it("never reports a disabled external server", async () => {
+    row.enabled = false;
+    row.game_server_node_id = null;
+    reachable = false;
+
+    for (let minute = 0; minute < 4; minute++) {
+      await service.pingDedicatedServer("server-1");
+      minutes(1);
+    }
+
+    expect(alerts()).toBe(0);
+  });
+
+  it("starts the clock over after a gap in pings", async () => {
+    reachable = false;
+
+    await service.pingDedicatedServer("server-1");
+    minutes(5);
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+    expect(alerts()).toBe(0);
+
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+    expect(alerts()).toBe(1);
+  });
+
+  it("tries the alert again when sending it failed", async () => {
+    reachable = false;
+    notifications.send.mockRejectedValueOnce(new Error("discord down"));
+
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await expect(service.pingDedicatedServer("server-1")).rejects.toThrow(
+      "discord down",
+    );
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+    minutes(1);
+    await service.pingDedicatedServer("server-1");
+
+    expect(alerts()).toBe(2);
+  });
+
+  it("forgets the streak of a server that is removed", async () => {
+    (service as any).apps = { deleteNamespacedDeployment: jest.fn() };
+    reachable = false;
+
+    await service.pingDedicatedServer("server-1");
+    await service.removeDedicatedServer("server-1");
+
+    expect(
+      await redis.hget("dedicated-servers:unreachable", "server-1"),
+    ).toBeNull();
+  });
+});
+
 describe("DedicatedServersService.pluginInstallEnvironment", () => {
   const installs = (type: string, game = "cs2") =>
     Object.fromEntries(