diff --git a/hasura/metadata/databases/default/tables/public_chat_message_deletions.yaml b/hasura/metadata/databases/default/tables/public_chat_message_deletions.yaml index 310fe26c..3b7d1e7e 100644 --- a/hasura/metadata/databases/default/tables/public_chat_message_deletions.yaml +++ b/hasura/metadata/databases/default/tables/public_chat_message_deletions.yaml @@ -22,6 +22,8 @@ select_permissions: - source - deleted_by_steam_id - deleted_at + - attachments + - gif filter: {} allow_aggregations: true comment: Evidence of moderated website chat. Written only by the API. @@ -38,6 +40,8 @@ select_permissions: - source - deleted_by_steam_id - deleted_at + - attachments + - gif filter: room_type: _neq: organizers diff --git a/hasura/metadata/databases/default/tables/public_settings.yaml b/hasura/metadata/databases/default/tables/public_settings.yaml index 734b3d21..13d03dfa 100644 --- a/hasura/metadata/databases/default/tables/public_settings.yaml +++ b/hasura/metadata/databases/default/tables/public_settings.yaml @@ -19,6 +19,7 @@ select_permissions: name: _nin: - web_push_private_key + - giphy_api_key comment: "" - role: guest permission: @@ -33,7 +34,6 @@ update_permissions: - role: administrator permission: columns: - - name - value filter: {} check: {} diff --git a/hasura/migrations/default/1890000000200_chat_attachments/down.sql b/hasura/migrations/default/1890000000200_chat_attachments/down.sql new file mode 100644 index 00000000..bbe4e7f0 --- /dev/null +++ b/hasura/migrations/default/1890000000200_chat_attachments/down.sql @@ -0,0 +1,16 @@ +DROP TRIGGER IF EXISTS tad_direct_messages ON public.direct_messages; +DROP FUNCTION IF EXISTS public.tad_direct_messages(); + +-- The boot step only re-applies a triggers file whose digest changed, so +-- forget this one or a later up would never recreate it. +DELETE FROM migration_hashes.hashes + WHERE name = 'hasura/triggers/direct_messages'; + +ALTER TABLE public.direct_messages + DROP COLUMN IF EXISTS attachments, + DROP COLUMN IF EXISTS gif; + +DROP TABLE IF EXISTS public.chat_attachments; + +DELETE FROM public.settings + WHERE name IN ('chat_attachment_max_mb', 'giphy_api_key'); diff --git a/hasura/migrations/default/1890000000200_chat_attachments/up.sql b/hasura/migrations/default/1890000000200_chat_attachments/up.sql new file mode 100644 index 00000000..170b1c52 --- /dev/null +++ b/hasura/migrations/default/1890000000200_chat_attachments/up.sql @@ -0,0 +1,46 @@ +CREATE TABLE IF NOT EXISTS public.chat_attachments ( + id uuid PRIMARY KEY DEFAULT gen_random_uuid(), + -- Kept when the player goes: the row is what the file is swept by. + uploader_steam_id bigint REFERENCES public.players (steam_id) + ON UPDATE CASCADE ON DELETE SET NULL, + room_type text NOT NULL, + room_id text NOT NULL, + storage_prefix text NOT NULL, + file_name text NOT NULL, + mime_type text NOT NULL, + size bigint NOT NULL CHECK (size > 0), + width integer, + height integer, + duration_ms integer, + poster_mime_type text, + -- Set until the multipart upload completes. + upload_id text, + uploaded_at timestamptz, + message_id uuid, + sent_at timestamptz, + -- NULL only for a sent direct message's file, which goes with its message. + expires_at timestamptz, + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS chat_attachments_expires_at_idx + ON public.chat_attachments (expires_at) + WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS chat_attachments_message_id_idx + ON public.chat_attachments (message_id) + WHERE message_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS chat_attachments_pending_idx + ON public.chat_attachments (uploader_steam_id) + WHERE message_id IS NULL; + +CREATE INDEX IF NOT EXISTS chat_attachments_room_idx + ON public.chat_attachments (room_type, room_id); + +CREATE INDEX IF NOT EXISTS chat_attachments_storage_prefix_idx + ON public.chat_attachments (storage_prefix text_pattern_ops); + +ALTER TABLE public.direct_messages + ADD COLUMN IF NOT EXISTS attachments jsonb, + ADD COLUMN IF NOT EXISTS gif jsonb; diff --git a/hasura/migrations/default/1890000000210_chat_attachment_evidence_and_usage/down.sql b/hasura/migrations/default/1890000000210_chat_attachment_evidence_and_usage/down.sql new file mode 100644 index 00000000..1a5c0124 --- /dev/null +++ b/hasura/migrations/default/1890000000210_chat_attachment_evidence_and_usage/down.sql @@ -0,0 +1,11 @@ +ALTER TABLE public.chat_message_deletions + DROP COLUMN IF EXISTS attachments, + DROP COLUMN IF EXISTS gif; + +DROP TABLE IF EXISTS public.chat_attachment_usage; + +ALTER TABLE public.chat_attachments + DROP COLUMN IF EXISTS deleted_at; + +DELETE FROM public.settings + WHERE name IN ('chat_attachment_daily_mb', 'giphy_hourly_limit'); diff --git a/hasura/migrations/default/1890000000210_chat_attachment_evidence_and_usage/up.sql b/hasura/migrations/default/1890000000210_chat_attachment_evidence_and_usage/up.sql new file mode 100644 index 00000000..22dc4e17 --- /dev/null +++ b/hasura/migrations/default/1890000000210_chat_attachment_evidence_and_usage/up.sql @@ -0,0 +1,24 @@ +-- Set when a group room's message is deleted: the file is kept as evidence, +-- for staff only, until it expires. +ALTER TABLE public.chat_attachments + ADD COLUMN IF NOT EXISTS deleted_at timestamptz; + +-- What a player uploaded, kept apart from the files so removing a file does +-- not give its bytes back to the daily allowance. +CREATE TABLE IF NOT EXISTS public.chat_attachment_usage ( + id bigserial PRIMARY KEY, + steam_id bigint NOT NULL REFERENCES public.players (steam_id) + ON UPDATE CASCADE ON DELETE CASCADE, + bytes bigint NOT NULL CHECK (bytes > 0), + created_at timestamptz NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS chat_attachment_usage_steam_id_created_at_idx + ON public.chat_attachment_usage (steam_id, created_at); + +CREATE INDEX IF NOT EXISTS chat_attachment_usage_created_at_idx + ON public.chat_attachment_usage (created_at); + +ALTER TABLE public.chat_message_deletions + ADD COLUMN IF NOT EXISTS attachments jsonb, + ADD COLUMN IF NOT EXISTS gif jsonb; diff --git a/hasura/triggers/direct_messages.sql b/hasura/triggers/direct_messages.sql new file mode 100644 index 00000000..339b8bfe --- /dev/null +++ b/hasura/triggers/direct_messages.sql @@ -0,0 +1,23 @@ +-- Whatever deletes a direct message -- its author, the retention sweep, its +-- author's account going -- its files are due. The sweep removes them from +-- storage before it forgets the row. +CREATE OR REPLACE FUNCTION public.tad_direct_messages() RETURNS TRIGGER + LANGUAGE plpgsql + AS $$ +BEGIN + UPDATE public.chat_attachments a + SET expires_at = now() + FROM deleted d + WHERE a.message_id = d.id + AND a.room_type = 'direct' + AND (a.expires_at IS NULL OR a.expires_at > now()); + + RETURN NULL; +END; +$$; + +DROP TRIGGER IF EXISTS tad_direct_messages ON public.direct_messages; +CREATE TRIGGER tad_direct_messages + AFTER DELETE ON public.direct_messages + REFERENCING OLD TABLE AS deleted + FOR EACH STATEMENT EXECUTE FUNCTION public.tad_direct_messages(); diff --git a/src/chat/chat-attachments-abort.spec.ts b/src/chat/chat-attachments-abort.spec.ts new file mode 100644 index 00000000..3f985d46 --- /dev/null +++ b/src/chat/chat-attachments-abort.spec.ts @@ -0,0 +1,154 @@ +import { + createServer, + request as httpRequest, + IncomingMessage, + Server, +} from "http"; +import { AddressInfo } from "net"; +import { Readable, Writable } from "stream"; +import { ChatAttachmentsService } from "./chat-attachments.service"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; + +const ID = "0b7d6c1e-1111-4a2b-9c3d-000000000001"; +const PART = ChatAttachmentsService.PART_SIZE; + +// A real socket that goes away mid-body, against storage that does what the +// AWS SDK does with a body: pipe it, and listen for nothing else on it. +describe("ChatAttachmentsService, a part whose sender goes away", () => { + let server: Server; + let service: ChatAttachmentsService; + let sends: Array<{ signal?: AbortSignal; settled: Promise }>; + let queries: string[]; + let arrived: () => void; + let started: Promise; + let result: Promise; + + const row = { + id: ID, + uploader_steam_id: "1", + room_type: "matchmaking", + room_id: "lobby-1", + storage_prefix: `chat-attachments/rooms/2026-10-02/${ID}/`, + file_name: "clip.mp4", + mime_type: "video/mp4", + size: String(PART * 3), + width: null, + height: null, + duration_ms: null, + poster_mime_type: null, + upload_id: "upload-1", + message_id: null, + deleted_at: null, + }; + + const s3 = { + uploadPart: jest.fn( + ( + _key: string, + _uploadId: string, + _part: number, + body: Readable, + _length: number, + signal?: AbortSignal, + ) => { + const settled = new Promise((resolve, reject) => { + const sink = new Writable({ + write(_chunk, _encoding, callback) { + arrived(); + callback(); + }, + }); + + body.pipe(sink); + sink.on("finish", () => resolve()); + signal?.addEventListener("abort", () => + reject(new Error("request aborted")), + ); + }); + + sends.push({ signal, settled: settled.catch(() => {}) }); + + return settled; + }, + ), + }; + + const postgres = { + query: jest.fn(async (sql: string) => { + queries.push(sql); + return sql.includes("upload_id IS NOT NULL") ? [row] : []; + }), + }; + + beforeEach(async () => { + sends = []; + queries = []; + started = new Promise((resolve) => { + arrived = resolve; + }); + + service = new ChatAttachmentsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + postgres as any, + s3 as any, + ); + + server = createServer((request: IncomingMessage) => { + result = service.uploadPart("1", ID, 2, request, PART); + }); + + await new Promise((resolve) => server.listen(0, resolve)); + }); + + afterEach(async () => { + await new Promise((resolve) => server.close(resolve)); + }); + + const abortMidBody = async () => { + const { port } = server.address() as AddressInfo; + + const client = httpRequest({ + port, + method: "PUT", + headers: { + "content-type": "application/octet-stream", + "content-length": String(PART), + }, + }); + client.on("error", () => {}); + + client.write(Buffer.alloc(300 * 1024)); + await started; + client.destroy(); + + return await result; + }; + + it("survives, refuses the part, and lets go of the request to storage", async () => { + await expect(abortMidBody()).resolves.toBe(ChatErrorCode.Invalid); + + expect(sends).toHaveLength(1); + expect(sends[0].signal?.aborted).toBe(true); + await sends[0].settled; + }); + + it("frees the player's upload slots for the next try", async () => { + await abortMidBody(); + + expect((service as any).uploadsInFlight).toBe(0); + expect((service as any).uploadsByPlayer.size).toBe(0); + }); + + it("leaves the upload for the sweep to abort", async () => { + await abortMidBody(); + + expect( + queries.filter( + (sql) => + sql.includes("DELETE FROM public.chat_attachments") || + sql.includes("SET expires_at") || + sql.includes("SET upload_id = NULL"), + ), + ).toEqual([]); + }); +}); diff --git a/src/chat/chat-attachments-stall.spec.ts b/src/chat/chat-attachments-stall.spec.ts new file mode 100644 index 00000000..8a63e6bb --- /dev/null +++ b/src/chat/chat-attachments-stall.spec.ts @@ -0,0 +1,103 @@ +import { createServer, IncomingMessage, Server, ServerResponse } from "http"; +import { AddressInfo } from "net"; +import { Readable } from "stream"; +import { ChatAttachmentsService } from "./chat-attachments.service"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; +import { S3Service } from "../s3/s3.service"; + +const ID = "0b7d6c1e-1111-4a2b-9c3d-000000000001"; +const PART = ChatAttachmentsService.PART_SIZE; +const IDLE_MS = 400; + +// A store that takes the whole part and then never answers, through the real +// S3 client: the player's upload slots must not wait on it. +describe("ChatAttachmentsService, a store that never answers a part", () => { + let server: Server; + let held: ServerResponse[]; + let s3: S3Service; + let service: ChatAttachmentsService; + const originalIdle = (S3Service as any).PART_IDLE_TIMEOUT_MS; + + const row = { + id: ID, + uploader_steam_id: "1", + room_type: "matchmaking", + room_id: "lobby-1", + storage_prefix: `chat-attachments/rooms/2026-10-02/${ID}/`, + file_name: "clip.mp4", + mime_type: "video/mp4", + size: String(PART * 3), + width: null, + height: null, + duration_ms: null, + poster_mime_type: null, + upload_id: "upload-1", + message_id: null, + deleted_at: null, + }; + + beforeAll(async () => { + held = []; + server = createServer((request: IncomingMessage, response) => { + request.resume(); + request.on("end", () => held.push(response)); + }); + await new Promise((resolve) => + server.listen(0, "127.0.0.1", resolve), + ); + + (S3Service as any).PART_IDLE_TIMEOUT_MS = IDLE_MS; + + const { port } = server.address() as AddressInfo; + s3 = new S3Service( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + { + get: () => ({ + key: "access-key", + secret: "secret-key", + bucket: "5stack", + endpoint: "127.0.0.1", + port: String(port), + useSSL: false, + region: "us-east-1", + forcePathStyle: true, + }), + } as any, + ); + + service = new ChatAttachmentsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + { + query: jest.fn(async (sql: string) => + sql.includes("upload_id IS NOT NULL") ? [row] : [], + ), + } as any, + s3, + ); + }); + + afterAll(async () => { + (S3Service as any).PART_IDLE_TIMEOUT_MS = originalIdle; + s3.onModuleDestroy(); + for (const response of held) { + response.destroy(); + } + await new Promise((resolve) => server.close(resolve)); + }); + + it("gives up once the store goes quiet, and frees the player's slots", async () => { + const started = Date.now(); + + await expect( + service.uploadPart("1", ID, 2, Readable.from([Buffer.alloc(PART)]), PART), + ).resolves.toBe(ChatErrorCode.Unavailable); + + expect(Date.now() - started).toBeLessThan(IDLE_MS + 4_000); + expect((service as any).uploadsInFlight).toBe(0); + expect((service as any).uploadsByPlayer.size).toBe(0); + }, 15_000); + + it("waits on a store that is quiet for less than the idle limit", () => { + expect(originalIdle).toBeGreaterThanOrEqual(30_000); + }); +}); diff --git a/src/chat/chat-attachments.service.spec.ts b/src/chat/chat-attachments.service.spec.ts new file mode 100644 index 00000000..5d0f716f --- /dev/null +++ b/src/chat/chat-attachments.service.spec.ts @@ -0,0 +1,1236 @@ +import { PassThrough, Readable } from "stream"; +import { ChatAttachmentsService } from "./chat-attachments.service"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; +import { ChatLobbyType } from "./enums/ChatLobbyTypes"; + +const MB = 1024 * 1024; + +const header = (...parts: Array) => + Buffer.concat( + parts.map((part) => + typeof part === "string" + ? Buffer.from(part, "latin1") + : Buffer.from(part), + ), + ); + +const PNG = header([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], "IHDR"); +const JPEG = header([0xff, 0xd8, 0xff, 0xe0], "JFIF"); +const GIF = header("GIF89a", [0, 0, 0, 0]); +const WEBP = header("RIFF", [0, 0, 0, 0], "WEBPVP8 "); +const MP4 = header([0, 0, 0, 0x20], "ftypisom", [0, 0, 2, 0]); +const MOV = header([0, 0, 0, 0x14], "ftypqt ", [0, 0, 0, 0]); +const OLD_MOV = header([0, 0, 0, 0x08], "wide", [0, 0, 0, 0]); +const WEBM = header([0x1a, 0x45, 0xdf, 0xa3], [0x9f, 0x42, 0x86, 0x81]); +const HTML = header(""); +const SVG = header(''); + +const u32be = (value: number) => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(value); + return bytes; +}; + +const u16be = (value: number) => { + const bytes = Buffer.alloc(2); + bytes.writeUInt16BE(value); + return bytes; +}; + +const u16le = (value: number) => { + const bytes = Buffer.alloc(2); + bytes.writeUInt16LE(value); + return bytes; +}; + +const u24le = (value: number) => + Buffer.from([value & 0xff, (value >> 8) & 0xff, (value >> 16) & 0xff]); + +const png = (width: number, height: number) => + Buffer.concat([ + PNG.subarray(0, 8), + u32be(13), + Buffer.from("IHDR"), + u32be(width), + u32be(height), + Buffer.alloc(32), + ]); + +const jpeg = (width: number, height: number) => + Buffer.concat([ + Buffer.from([0xff, 0xd8]), + Buffer.from([0xff, 0xe1]), + u16be(2 + 300), + Buffer.alloc(300), + Buffer.from([0xff, 0xc2]), + u16be(11), + Buffer.from([8]), + u16be(height), + u16be(width), + Buffer.alloc(16), + ]); + +const gif = (width: number, height: number) => + Buffer.concat([ + Buffer.from("GIF89a"), + u16le(width), + u16le(height), + Buffer.alloc(16), + ]); + +const webpVp8x = (width: number, height: number) => + Buffer.concat([ + Buffer.from("RIFF"), + Buffer.alloc(4), + Buffer.from("WEBPVP8X"), + u32be(10), + Buffer.alloc(4), + u24le(width - 1), + u24le(height - 1), + Buffer.alloc(16), + ]); + +const webpVp8 = (width: number, height: number) => + Buffer.concat([ + Buffer.from("RIFF"), + Buffer.alloc(4), + Buffer.from("WEBPVP8 "), + Buffer.alloc(4), + Buffer.from([0, 0, 0, 0x9d, 0x01, 0x2a]), + u16le(width), + u16le(height), + Buffer.alloc(16), + ]); + +const webpVp8l = (width: number, height: number) => { + const bits = (width - 1) | ((height - 1) << 14); + const packed = Buffer.alloc(4); + packed.writeUInt32LE(bits >>> 0); + return Buffer.concat([ + Buffer.from("RIFF"), + Buffer.alloc(4), + Buffer.from("WEBPVP8L"), + Buffer.alloc(4), + Buffer.from([0x2f]), + packed, + Buffer.alloc(16), + ]); +}; + +describe("ChatAttachmentsService", () => { + describe("which rooms take attachments", () => { + it.each([ + ChatLobbyType.MatchMaking, + ChatLobbyType.Tournament, + ChatLobbyType.Organizer, + ChatLobbyType.Draft, + ChatLobbyType.Direct, + ])("allows %s", (type) => { + expect(ChatAttachmentsService.allowsAttachments(type)).toBe(true); + }); + + // Both are relayed into the CS2 server, which can only show text. + it.each([ChatLobbyType.Match, ChatLobbyType.MatchTeam, ChatLobbyType.Team])( + "keeps %s text-only", + (type) => { + expect(ChatAttachmentsService.allowsAttachments(type)).toBe(false); + }, + ); + }); + + describe("upload limits", () => { + const upload = (overrides: Record = {}) => ({ + name: "smoke.png", + size: 1024, + mime_type: "image/png", + ...overrides, + }); + + it.each([ + "image/png", + "image/jpeg", + "image/webp", + "image/gif", + "video/mp4", + "video/webm", + "video/quicktime", + ])("accepts %s", (mimeType) => { + expect( + ChatAttachmentsService.uploadRefusal( + upload({ mime_type: mimeType }), + 100 * MB, + ), + ).toBeNull(); + }); + + it.each([ + "image/svg+xml", + "text/html", + "application/pdf", + "audio/mpeg", + "", + undefined, + ])("refuses %s", (mimeType) => { + expect( + ChatAttachmentsService.uploadRefusal( + upload({ mime_type: mimeType }), + 100 * MB, + ), + ).toBe(ChatErrorCode.UnsupportedType); + }); + + it("holds every file to the operator's limit", () => { + expect( + ChatAttachmentsService.uploadRefusal( + upload({ size: 100 * MB }), + 100 * MB, + ), + ).toBeNull(); + expect( + ChatAttachmentsService.uploadRefusal( + upload({ size: 100 * MB + 1 }), + 100 * MB, + ), + ).toBe(ChatErrorCode.TooLarge); + }); + + it.each([0, -1, 1.5, "10", null, Number.NaN])( + "refuses a size of %p", + (size) => { + expect( + ChatAttachmentsService.uploadRefusal(upload({ size }), 100 * MB), + ).toBe(ChatErrorCode.Invalid); + }, + ); + + it("caps four attachments to a message", () => { + expect(ChatAttachmentsService.MAX_PER_MESSAGE).toBe(4); + }); + }); + + describe("the operator's limit", () => { + it.each([ + [null, 100 * MB], + ["", 100 * MB], + ["not a number", 100 * MB], + ["0", 100 * MB], + ["25", 25 * MB], + ["100000", 1024 * MB], + ])("reads %p as %p bytes", (value, bytes) => { + expect(ChatAttachmentsService.maxFileBytesFrom(value)).toBe(bytes); + }); + }); + + describe("file names", () => { + it.each([ + ["clutch.mp4", "clutch.mp4"], + ["../../etc/passwd.png", "passwd.png"], + ["C:\\Users\\me\\smoke b.png", "smoke b.png"], + ["line\nbreak\u0000.png", "linebreak.png"], + ["", "file"], + [null, "file"], + [42, "file"], + ])("stores %p as %p", (raw, name) => { + expect(ChatAttachmentsService.fileName(raw)).toBe(name); + }); + + it("keeps the extension of a name it has to shorten", () => { + const name = ChatAttachmentsService.fileName(`${"a".repeat(400)}.webm`); + + expect(name.length).toBeLessThanOrEqual(120); + expect(name.endsWith(".webm")).toBe(true); + }); + }); + + describe("content sniffing", () => { + it.each([ + [PNG, "image/png"], + [JPEG, "image/jpeg"], + [GIF, "image/gif"], + [WEBP, "image/webp"], + [MP4, "video/mp4"], + [MOV, "video/quicktime"], + [OLD_MOV, "video/quicktime"], + [WEBM, "video/webm"], + [HTML, null], + [SVG, null], + [Buffer.alloc(0), null], + ])("reads %#", (bytes, mimeType) => { + expect(ChatAttachmentsService.sniff(bytes)).toBe(mimeType); + }); + + // A browser names the type from the extension, so a jpeg saved as .png is + // still an image -- but nothing declared an image may turn out a video. + it("serves what the bytes are, within the kind that was declared", () => { + expect(ChatAttachmentsService.contentType("image/png", JPEG)).toBe( + "image/jpeg", + ); + expect(ChatAttachmentsService.contentType("video/quicktime", MP4)).toBe( + "video/mp4", + ); + expect(ChatAttachmentsService.contentType("image/png", MP4)).toBeNull(); + expect(ChatAttachmentsService.contentType("video/mp4", PNG)).toBeNull(); + expect(ChatAttachmentsService.contentType("image/png", HTML)).toBeNull(); + }); + + it("only takes a still image as a video's poster", () => { + expect(ChatAttachmentsService.posterType(WEBP)).toBe("image/webp"); + expect(ChatAttachmentsService.posterType(PNG)).toBe("image/png"); + expect(ChatAttachmentsService.posterType(JPEG)).toBe("image/jpeg"); + expect(ChatAttachmentsService.posterType(GIF)).toBeNull(); + expect(ChatAttachmentsService.posterType(HTML)).toBeNull(); + }); + }); + + describe("image size", () => { + it.each([ + ["a png", png(800, 600), "image/png"], + ["a progressive jpeg past its exif", jpeg(800, 600), "image/jpeg"], + ["a gif", gif(800, 600), "image/gif"], + ["an extended webp", webpVp8x(800, 600), "image/webp"], + ["a lossy webp", webpVp8(800, 600), "image/webp"], + ["a lossless webp", webpVp8l(800, 600), "image/webp"], + ])("reads %s from its header", (_, bytes, mimeType) => { + expect(ChatAttachmentsService.imageSize(bytes, mimeType)).toEqual({ + width: 800, + height: 600, + }); + }); + + it("gives up on a header it cannot read", () => { + expect(ChatAttachmentsService.imageSize(JPEG, "image/jpeg")).toBeNull(); + expect( + ChatAttachmentsService.imageSize(Buffer.alloc(4), "image/png"), + ).toBeNull(); + }); + + it("caps an image at 40 megapixels", () => { + expect(ChatAttachmentsService.MAX_PIXELS).toBe(40_000_000); + }); + }); + + describe("download names", () => { + it.each([ + ["smoke.png", "image/jpeg", "smoke.jpg"], + ["clutch.MOV", "video/quicktime", "clutch.mov"], + ["clip", "video/mp4", "clip.mp4"], + ["evil.html", "image/png", "evil.png"], + ["", "image/webp", "file.webp"], + ])("serves %p as %p bytes under %p", (name, mimeType, served) => { + expect(ChatAttachmentsService.downloadName(name, mimeType)).toBe(served); + }); + }); + + describe("the daily allowance", () => { + it.each([ + [null, 1024 * MB], + ["", 1024 * MB], + ["0", 1024 * MB], + ["500", 500 * MB], + ])("reads %p as %p bytes", (value, bytes) => { + expect(ChatAttachmentsService.dailyQuotaBytesFrom(value)).toBe(bytes); + }); + }); + + describe("uploading a part", () => { + const ID = "0b7d6c1e-1111-4a2b-9c3d-000000000001"; + const PART = ChatAttachmentsService.PART_SIZE; + + let stored: Record | null; + let updates: Array<{ sql: string; bindings: any[] }>; + let sent: Array<{ part: number; bytes: Buffer; length: number }>; + let gate: Promise | null; + + const upload = (overrides: Record = {}) => ({ + id: ID, + uploader_steam_id: "1", + room_type: "matchmaking", + room_id: "lobby-1", + storage_prefix: `chat-attachments/rooms/2026-10-02/${ID}/`, + file_name: "smoke.png", + mime_type: "image/png", + size: String(PART + 100), + width: 10, + height: 10, + duration_ms: null, + poster_mime_type: null, + upload_id: "upload-1", + message_id: null, + deleted_at: null, + ...overrides, + }); + + const postgres = { + query: jest.fn(async (sql: string, bindings: any[] = []) => { + if (sql.includes("upload_id IS NOT NULL")) { + return stored ? [stored] : []; + } + + updates.push({ sql, bindings }); + return []; + }), + }; + + const s3 = { + uploadPart: jest.fn( + async ( + _key: string, + _uploadId: string, + part: number, + body: Readable, + length: number, + ) => { + const chunks: Buffer[] = []; + for await (const chunk of body) { + chunks.push(chunk as Buffer); + } + sent.push({ part, bytes: Buffer.concat(chunks), length }); + await gate; + }, + ), + abortMultipartUpload: jest.fn(), + removePrefixStrictly: jest.fn(), + }; + + const service = () => + new ChatAttachmentsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + postgres as any, + s3 as any, + ); + + // Hands the bytes over in small chunks, as a socket would, and counts how + // many were ever asked for. + const body = (bytes: Buffer) => { + let offset = 0; + const stream = new Readable({ + read() { + stream.reads++; + if (offset >= bytes.length) { + this.push(null); + return; + } + this.push(bytes.subarray(offset, offset + 64 * 1024)); + offset += 64 * 1024; + }, + }) as Readable & { reads: number }; + stream.reads = 0; + return stream; + }; + + const firstPart = (head: Buffer) => + Buffer.concat([head, Buffer.alloc(PART - head.length)]); + + beforeEach(() => { + jest.clearAllMocks(); + stored = upload(); + updates = []; + sent = []; + gate = null; + }); + + it("checks the upload is the player's before reading a byte of it", async () => { + stored = null; + const part = body(firstPart(png(10, 10))); + + await expect(service().uploadPart("1", ID, 1, part, PART)).resolves.toBe( + ChatErrorCode.NotFound, + ); + + expect(part.reads).toBe(0); + expect(s3.uploadPart).not.toHaveBeenCalled(); + }); + + it("refuses a part that says it is the wrong length, without reading it", async () => { + const part = body(firstPart(png(10, 10))); + + await expect( + service().uploadPart("1", ID, 1, part, PART - 1), + ).resolves.toBe(ChatErrorCode.Invalid); + + expect(part.reads).toBe(0); + }); + + it("streams the part to storage with its length", async () => { + const bytes = firstPart(png(10, 10)); + + await expect( + service().uploadPart("1", ID, 1, body(bytes), PART), + ).resolves.toBeNull(); + + expect(s3.uploadPart.mock.calls[0][3]).toBeInstanceOf(Readable); + expect(sent).toEqual([{ part: 1, bytes, length: PART }]); + }); + + it("refuses a part that runs past the length it gave", async () => { + stored = upload({ size: String(PART + 100) }); + + await expect( + service().uploadPart("1", ID, 2, body(Buffer.alloc(200)), 100), + ).resolves.toBe(ChatErrorCode.Invalid); + }); + + it("refuses a part that stops short of the length it gave", async () => { + await expect( + service().uploadPart("1", ID, 2, body(Buffer.alloc(50)), 100), + ).resolves.toBe(ChatErrorCode.Invalid); + }); + + it("checks the first part's bytes before sending any of it on", async () => { + await expect( + service().uploadPart("1", ID, 1, body(firstPart(HTML)), PART), + ).resolves.toBe(ChatErrorCode.UnsupportedType); + + expect(s3.uploadPart).not.toHaveBeenCalled(); + }); + + it("refuses an image over the pixel cap, whatever size it claimed", async () => { + await expect( + service().uploadPart( + "1", + ID, + 1, + body(firstPart(png(10_000, 5_000))), + PART, + ), + ).resolves.toBe(ChatErrorCode.TooLarge); + + expect(s3.uploadPart).not.toHaveBeenCalled(); + }); + + // A GIF's frames carry their own size, and a decoder sizes its buffer + // from them, whatever the screen size at the top says. + // A decoder only grows its canvas for the first frame, so that frame has + // to be seen; a long comment can push it past what is read ahead. + it("refuses a GIF whose first frame is past what it reads ahead", async () => { + stored = upload({ mime_type: "image/gif" }); + const comment = Buffer.alloc(1_100_000); + const blocks: Buffer[] = []; + for (let at = 0; at < comment.length; at += 255) { + const block = comment.subarray(at, at + 255); + blocks.push(Buffer.from([block.length]), block); + } + const gif = Buffer.concat([ + Buffer.from("GIF89a"), + u16le(1), + u16le(1), + Buffer.from([0x00, 0x00, 0x00]), + Buffer.from([0x21, 0xfe]), + ...blocks, + Buffer.from([0x00]), + Buffer.from([0x2c]), + u16le(0), + u16le(0), + u16le(65535), + u16le(65535), + Buffer.from([0x00]), + ]); + + await expect( + service().uploadPart("1", ID, 1, body(firstPart(gif)), PART), + ).resolves.toBe(ChatErrorCode.UnsupportedType); + expect(s3.uploadPart).not.toHaveBeenCalled(); + }); + + it("refuses a GIF with a frame over the pixel cap, behind a small screen", async () => { + stored = upload({ mime_type: "image/gif" }); + const frame = Buffer.concat([ + Buffer.from("GIF89a"), + u16le(10), + u16le(10), + Buffer.from([0x00, 0x00, 0x00]), + Buffer.from([0x21, 0xf9, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00]), + Buffer.from([0x2c]), + u16le(0), + u16le(0), + u16le(10_000), + u16le(5_000), + Buffer.from([0x00]), + ]); + + await expect( + service().uploadPart("1", ID, 1, body(firstPart(frame)), PART), + ).resolves.toBe(ChatErrorCode.TooLarge); + }); + + it("keeps the size the image really is, not the one it claimed", async () => { + await service().uploadPart( + "1", + ID, + 1, + body(firstPart(png(800, 600))), + PART, + ); + + expect( + updates.find(({ sql }) => sql.includes("SET mime_type"))?.bindings, + ).toEqual([ID, "image/png", 800, 600]); + }); + + it("refuses an image whose size it cannot read", async () => { + await expect( + service().uploadPart("1", ID, 1, body(firstPart(PNG)), PART), + ).resolves.toBe(ChatErrorCode.UnsupportedType); + }); + + it("lets one player send two parts at a time, and answers the third busy", async () => { + stored = upload({ mime_type: "video/mp4", size: String(PART * 4) }); + let open = () => {}; + gate = new Promise((resolve) => { + open = resolve; + }); + const attachments = service(); + const part = () => body(Buffer.alloc(PART)); + + const first = attachments.uploadPart("1", ID, 2, part(), PART); + const second = attachments.uploadPart("1", ID, 3, part(), PART); + + await expect( + attachments.uploadPart("1", ID, 4, part(), PART), + ).resolves.toBe(ChatErrorCode.RateLimited); + + const someoneElse = attachments.uploadPart("2", ID, 4, part(), PART); + + open(); + + await expect(Promise.all([first, second, someoneElse])).resolves.toEqual([ + null, + null, + null, + ]); + + await expect( + attachments.uploadPart("1", ID, 4, part(), PART), + ).resolves.toBeNull(); + }); + + it("caps how many parts one process takes in at once", () => { + expect(ChatAttachmentsService.MAX_UPLOADS_PER_PLAYER).toBe(2); + expect(ChatAttachmentsService.MAX_UPLOADS_PER_PROCESS).toBe(16); + }); + }); + + describe("parts", () => { + const PART = ChatAttachmentsService.PART_SIZE; + + // Cloudflare refuses a request body over 100 MB, so no single part can be. + it("keeps every part well under Cloudflare's body cap", () => { + expect(PART).toBeLessThanOrEqual(64 * MB); + expect(PART).toBeGreaterThanOrEqual(5 * MB); + }); + + it.each([ + [1, 1], + [PART, 1], + [PART + 1, 2], + [100 * MB, Math.ceil((100 * MB) / PART)], + ])("splits %p bytes into %p part(s)", (size, parts) => { + expect(ChatAttachmentsService.partCount(size)).toBe(parts); + }); + + it("expects full parts and a short last one", () => { + const size = PART * 2 + 10; + + expect(ChatAttachmentsService.partLength(size, 1)).toBe(PART); + expect(ChatAttachmentsService.partLength(size, 2)).toBe(PART); + expect(ChatAttachmentsService.partLength(size, 3)).toBe(10); + }); + + it.each([0, 4, -1, 1.5, Number.NaN])( + "has no part %p of a three part file", + (part) => { + expect( + ChatAttachmentsService.partLength(PART * 2 + 10, part), + ).toBeNull(); + }, + ); + }); + + describe("expiry", () => { + const now = new Date("2026-10-02T12:00:00.000Z"); + + it("gives an upload that is never sent a day", () => { + expect(ChatAttachmentsService.pendingExpiry(now).toISOString()).toBe( + "2026-10-03T12:00:00.000Z", + ); + }); + + it("lets a room's file live as long as the room keeps its messages", () => { + const expiresAt = ChatAttachmentsService.expiresOnSend( + ChatLobbyType.Tournament, + 7 * 24 * 60 * 60, + now, + ); + + expect(expiresAt.getTime()).toBe( + now.getTime() + + 7 * 24 * 60 * 60 * 1000 + + ChatAttachmentsService.EXPIRY_GRACE_MS, + ); + }); + + it("never times out a direct message's file: it goes with the message", () => { + expect( + ChatAttachmentsService.expiresOnSend(ChatLobbyType.Direct, 3600, now), + ).toBeNull(); + }); + }); + + describe("storage layout", () => { + const id = "6f1c0e2a-6a4b-4d2f-9a51-0d7f3b1c2e3d"; + + it("files a room's upload under the day it was made", () => { + expect( + ChatAttachmentsService.storagePrefix( + ChatLobbyType.MatchMaking, + id, + new Date("2026-10-02T23:59:59.000Z"), + ), + ).toBe(`chat-attachments/rooms/2026-10-02/${id}/`); + }); + + it("keeps direct messages' uploads apart, they outlive any room", () => { + expect( + ChatAttachmentsService.storagePrefix( + ChatLobbyType.Direct, + id, + new Date("2026-10-02T00:00:00.000Z"), + ), + ).toBe(`chat-attachments/direct/2026-10-02/${id}/`); + }); + }); + + describe("who may see a file", () => { + const row = (overrides: Record = {}) => ({ + uploader_steam_id: "1", + message_id: "m-1", + room_type: "matchmaking", + deleted_at: null, + ...overrides, + }); + + const viewer = (steam_id: string, role = "user") => + ({ steam_id, role }) as any; + + it("always shows the uploader their own file", async () => { + const roomAccess = jest.fn().mockResolvedValue(false); + + await expect( + ChatAttachmentsService.canView( + row({ message_id: null }) as any, + viewer("1"), + roomAccess, + ), + ).resolves.toBe(true); + expect(roomAccess).not.toHaveBeenCalled(); + }); + + it("shows nobody else a file that was never sent", async () => { + const roomAccess = jest.fn().mockResolvedValue(true); + + await expect( + ChatAttachmentsService.canView( + row({ message_id: null }) as any, + viewer("2"), + roomAccess, + ), + ).resolves.toBe(false); + }); + + it("shows a sent file to whoever can open its room", async () => { + await expect( + ChatAttachmentsService.canView( + row() as any, + viewer("2"), + async () => true, + ), + ).resolves.toBe(true); + await expect( + ChatAttachmentsService.canView( + row() as any, + viewer("2"), + async () => false, + ), + ).resolves.toBe(false); + }); + + // Evidence: a deleted message's files stay for staff until they expire, + // and go dark for everyone in the room, its author included. + it("hides a deleted message's files from the room, its author too", async () => { + const deleted = row({ deleted_at: "2026-10-02T12:00:00.000Z" }) as any; + + await expect( + ChatAttachmentsService.canView(deleted, viewer("1"), async () => true), + ).resolves.toBe(false); + await expect( + ChatAttachmentsService.canView(deleted, viewer("2"), async () => true), + ).resolves.toBe(false); + }); + + it("still shows a deleted message's files to a moderator", async () => { + const roomAccess = jest.fn().mockResolvedValue(false); + + await expect( + ChatAttachmentsService.canView( + row({ deleted_at: "2026-10-02T12:00:00.000Z" }) as any, + viewer("9", "moderator"), + roomAccess, + ), + ).resolves.toBe(true); + expect(roomAccess).not.toHaveBeenCalled(); + }); + + it("keeps the organizers' deleted files from moderators, like its evidence", async () => { + const deleted = row({ + room_type: "organizers", + deleted_at: "2026-10-02T12:00:00.000Z", + }) as any; + + await expect( + ChatAttachmentsService.canView( + deleted, + viewer("9", "moderator"), + async () => false, + ), + ).resolves.toBe(false); + await expect( + ChatAttachmentsService.canView( + deleted, + viewer("9", "match_organizer"), + async () => false, + ), + ).resolves.toBe(true); + }); + + it("shows nothing to a request without a player", async () => { + await expect( + ChatAttachmentsService.canView( + row() as any, + undefined, + async () => true, + ), + ).resolves.toBe(false); + }); + }); + + describe("serving", () => { + const s3 = { + stat: jest.fn(async () => ({ size: 100, etag: "abc", metaData: {} })), + get: jest.fn(async () => Readable.from([Buffer.alloc(100)])), + getPartial: jest.fn(async () => Readable.from([Buffer.alloc(10)])), + }; + + const response = () => { + const headers: Record = {}; + const res = new PassThrough() as any; + res.statusCode = 200; + res.setHeader = (name: string, value: string) => { + headers[name.toLowerCase()] = value; + }; + res.status = (code: number) => { + res.statusCode = code; + return res; + }; + res.headers = headers; + res.resume(); + return res; + }; + + const service = () => + new ChatAttachmentsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + {} as any, + s3 as any, + ); + + beforeEach(() => { + jest.clearAllMocks(); + }); + + // A deleted message's file must stop showing the moment it is deleted, + // not an hour later from the browser's cache. + it("makes every view ask again, with an ETag to keep that cheap", async () => { + const res = response(); + + await service().stream( + "key", + "image/png", + "smoke.png", + { headers: {} } as any, + res, + ); + + expect(res.headers["cache-control"]).toBe("private, no-cache"); + expect(res.headers["etag"]).toBe('"abc"'); + expect(res.headers["etag"]).not.toMatch(/^W\//); + expect(res.statusCode).toBe(200); + }); + + it.each([['"abc"'], ['W/"abc"'], ['"old", "abc"'], ["*"]])( + "answers 304 to If-None-Match %s", + async (header) => { + const res = response(); + + await service().stream( + "key", + "image/png", + "smoke.png", + { headers: { "if-none-match": header } } as any, + res, + ); + + expect(res.statusCode).toBe(304); + expect(s3.get).not.toHaveBeenCalled(); + expect(s3.getPartial).not.toHaveBeenCalled(); + }, + ); + + it.each([['"old"'], ["abc"], [""]])( + "sends the file to If-None-Match %p", + async (header) => { + const res = response(); + + await service().stream( + "key", + "image/png", + "smoke.png", + { headers: { "if-none-match": header } } as any, + res, + ); + + expect(res.statusCode).toBe(200); + }, + ); + }); + + describe("ranges", () => { + it.each([ + ["bytes=0-99", 1000, { start: 0, end: 99 }], + ["bytes=900-", 1000, { start: 900, end: 999 }], + ["bytes=-100", 1000, { start: 900, end: 999 }], + ["bytes=0-5000", 1000, { start: 0, end: 999 }], + ["bytes=1000-", 1000, null], + ["bytes=5-1", 1000, null], + ["bytes=0-1,5-9", 1000, null], + ["items=0-1", 1000, null], + ])("reads %p of %p bytes", (value, size, range) => { + expect(ChatAttachmentsService.parseRange(value, size)).toEqual(range); + }); + }); + + describe("cleanup", () => { + const s3 = { + removePrefixStrictly: jest.fn(), + abortMultipartUpload: jest.fn(), + listPrefixes: jest.fn(), + listStream: jest.fn(), + }; + + let rows: Array<{ + id: string; + storage_prefix: string; + upload_id: string | null; + expired: boolean; + }>; + let order: string[]; + + const postgres = { + query: jest.fn(async (sql: string, bindings: any[] = []) => { + if (sql.includes("expires_at <= now()") && sql.includes("SELECT")) { + return rows.filter((row) => row.expired); + } + + if (sql.includes("DELETE FROM public.chat_attachments")) { + order.push(`delete:${bindings[0]}`); + rows = rows.filter((row) => row.id !== bindings[0]); + return []; + } + + if (sql.includes("SET upload_id = NULL")) { + order.push(`aborted:${bindings[0]}`); + rows = rows.map((row) => + row.id === bindings[0] ? { ...row, upload_id: null } : row, + ); + return []; + } + + if (sql.includes("LIKE")) { + return rows + .filter((row) => + row.storage_prefix.startsWith( + String(bindings[0]).replace(/%$/, ""), + ), + ) + .map(({ id }) => ({ id })); + } + + return []; + }), + }; + + const service = () => + new ChatAttachmentsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + postgres as any, + s3 as any, + ); + + const prefix = (day: string, id: string, scope = "rooms") => + `chat-attachments/${scope}/${day}/${id}/`; + + beforeEach(() => { + jest.clearAllMocks(); + order = []; + rows = []; + s3.removePrefixStrictly.mockImplementation(async (value: string) => { + order.push(`sweep:${value}`); + return 1; + }); + s3.abortMultipartUpload.mockImplementation(async (key: string) => { + order.push(`abort:${key}`); + }); + s3.listPrefixes.mockResolvedValue([]); + s3.listStream.mockImplementation(async function* () {}); + }); + + describe("expired files", () => { + it("sweeps every version of a file before forgetting it", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: null, + expired: true, + }, + { + id: "a-2", + storage_prefix: prefix("2026-10-01", "a-2"), + upload_id: null, + expired: false, + }, + ]; + + await expect(service().removeExpired()).resolves.toBe(1); + + expect(order).toEqual([ + `sweep:${prefix("2026-10-01", "a-1")}`, + "delete:a-1", + ]); + expect(rows.map(({ id }) => id)).toEqual(["a-2"]); + }); + + it("aborts an upload that never finished, or its parts stay billed", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: "upload-1", + expired: true, + }, + ]; + + await service().removeExpired(); + + expect(s3.abortMultipartUpload).toHaveBeenCalledWith( + `${prefix("2026-10-01", "a-1")}file`, + "upload-1", + ); + expect(order).toEqual([ + `abort:${prefix("2026-10-01", "a-1")}file`, + "aborted:a-1", + `sweep:${prefix("2026-10-01", "a-1")}`, + "delete:a-1", + ]); + }); + + it("keeps the row when the sweep fails, so the next run tries again", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: null, + expired: true, + }, + ]; + s3.removePrefixStrictly.mockRejectedValue(new Error("403 deleteFiles")); + + await expect(service().removeExpired()).resolves.toBe(0); + + expect(rows.map(({ id }) => id)).toEqual(["a-1"]); + }); + + it("keeps the row, and its upload, when the abort fails, to try again", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: "upload-1", + expired: true, + }, + ]; + s3.abortMultipartUpload.mockRejectedValue( + Object.assign(new Error("SlowDown"), { name: "SlowDown" }), + ); + + await expect(service().removeExpired()).resolves.toBe(0); + + expect(s3.removePrefixStrictly).not.toHaveBeenCalled(); + expect(rows.map(({ id }) => id)).toEqual(["a-1"]); + }); + + it("forgets the upload once it is aborted, so a failed sweep does not abort it twice", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: "upload-1", + expired: true, + }, + ]; + s3.removePrefixStrictly.mockRejectedValue(new Error("403 deleteFiles")); + + await service().removeExpired(); + + expect(order).toContain("aborted:a-1"); + expect(rows.map(({ id }) => id)).toEqual(["a-1"]); + }); + + // A row whose expiry moved on after it was picked up must survive. + it("only deletes a row that is still due and has no upload left", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: null, + expired: true, + }, + ]; + + await service().removeExpired(); + + const sql = postgres.query.mock.calls + .map(([statement]) => statement) + .find((statement) => + statement.includes("DELETE FROM public.chat_attachments"), + ); + + expect(sql).toMatch(/expires_at <= now\(\)/); + expect(sql).toMatch(/upload_id IS NULL/); + }); + + it("still sweeps a file whose upload was already gone", async () => { + rows = [ + { + id: "a-1", + storage_prefix: prefix("2026-10-01", "a-1"), + upload_id: "upload-1", + expired: true, + }, + ]; + s3.abortMultipartUpload.mockRejectedValue( + Object.assign(new Error("NoSuchUpload"), { name: "NoSuchUpload" }), + ); + + await expect(service().removeExpired()).resolves.toBe(1); + + expect(rows).toEqual([]); + }); + }); + + describe("the daily sweep", () => { + const now = new Date("2026-10-10T04:41:00.000Z"); + + it("drops a day nothing points at in one delete, without listing it", async () => { + s3.listPrefixes.mockImplementation(async (value: string) => + value === "chat-attachments/rooms/" + ? ["chat-attachments/rooms/2026-10-01/"] + : [], + ); + + await service().sweepOrphans(now); + + expect(s3.removePrefixStrictly).toHaveBeenCalledWith( + "chat-attachments/rooms/2026-10-01/", + ); + expect(s3.listStream).not.toHaveBeenCalled(); + }); + + it("removes only the files no row points at in a day still in use", async () => { + const day = "chat-attachments/direct/2026-09-01/"; + rows = [ + { + id: "kept", + storage_prefix: `${day}kept/`, + upload_id: null, + expired: false, + }, + ]; + s3.listPrefixes.mockImplementation(async (value: string) => + value === "chat-attachments/direct/" ? [day] : [], + ); + s3.listStream.mockImplementation(async function* () { + yield { name: `${day}kept/file`, size: 1 }; + yield { name: `${day}kept/poster`, size: 1 }; + yield { name: `${day}stray/file`, size: 1 }; + }); + + await service().sweepOrphans(now); + + expect(s3.listStream).toHaveBeenCalledWith(day); + expect(s3.removePrefixStrictly.mock.calls).toEqual([[`${day}stray/`]]); + }); + + it("leaves today and yesterday alone while uploads may still land", async () => { + s3.listPrefixes.mockImplementation(async (value: string) => + value === "chat-attachments/rooms/" + ? [ + "chat-attachments/rooms/2026-10-09/", + "chat-attachments/rooms/2026-10-10/", + ] + : [], + ); + + await service().sweepOrphans(now); + + expect(s3.removePrefixStrictly).not.toHaveBeenCalled(); + expect(s3.listStream).not.toHaveBeenCalled(); + }); + + it("carries on to the next day when one cannot be swept", async () => { + s3.listPrefixes.mockImplementation(async (value: string) => + value === "chat-attachments/rooms/" + ? [ + "chat-attachments/rooms/2026-10-01/", + "chat-attachments/rooms/2026-10-02/", + ] + : [], + ); + s3.removePrefixStrictly.mockImplementation(async (value: string) => { + if (value.includes("2026-10-01")) { + throw new Error("AccessDenied"); + } + return 1; + }); + + await expect(service().sweepOrphans(now)).resolves.toBe(1); + + expect(s3.removePrefixStrictly.mock.calls.map(([day]) => day)).toEqual([ + "chat-attachments/rooms/2026-10-01/", + "chat-attachments/rooms/2026-10-02/", + ]); + }); + + it("ignores anything under the prefix that is not a day", async () => { + s3.listPrefixes.mockImplementation(async (value: string) => + value === "chat-attachments/rooms/" + ? ["chat-attachments/rooms/../", "chat-attachments/rooms/x/"] + : [], + ); + + await service().sweepOrphans(now); + + expect(s3.removePrefixStrictly).not.toHaveBeenCalled(); + }); + }); + }); +}); diff --git a/src/chat/chat-attachments.service.ts b/src/chat/chat-attachments.service.ts new file mode 100644 index 00000000..f0f09e46 --- /dev/null +++ b/src/chat/chat-attachments.service.ts @@ -0,0 +1,1679 @@ +import { randomUUID } from "crypto"; +import { pipeline, Readable, Transform } from "stream"; +import { Injectable, Logger } from "@nestjs/common"; +import { Request, Response } from "express"; +import { PoolClient } from "pg"; +import { e_player_roles_enum } from "generated"; +import { isRoleAbove } from "src/utilities/isRoleAbove"; +import { PostgresService } from "../postgres/postgres.service"; +import { S3Service } from "../s3/s3.service"; +import { SystemSettingName } from "../system/enums/SystemSettingName"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; +import { ChatLobbyType } from "./enums/ChatLobbyTypes"; +import { ChatAttachment, ChatAttachmentKind } from "./types/ChatAttachment"; + +export interface ChatAttachmentUpload { + name?: unknown; + size?: unknown; + mime_type?: unknown; + width?: unknown; + height?: unknown; + duration_ms?: unknown; +} + +export interface ChatAttachmentClaim { + type: ChatLobbyType; + roomId: string; + steamId: string; + messageId: string; + expiresAt: Date | null; +} + +export interface ChatAttachmentRow { + id: string; + uploader_steam_id: string | null; + room_type: ChatLobbyType; + room_id: string; + storage_prefix: string; + file_name: string; + mime_type: string; + size: string; + width: number | null; + height: number | null; + duration_ms: number | null; + poster_mime_type: string | null; + upload_id: string | null; + message_id: string | null; + deleted_at: string | null; +} + +export interface ChatAttachmentViewer { + steam_id: string; + role: e_player_roles_enum; +} + +type Removable = Pick; + +@Injectable() +export class ChatAttachmentsService { + public static readonly MAX_PER_MESSAGE = 4; + + // Each part is its own request through Cloudflare, which refuses a body over + // 100 MB. S3 refuses a part under 5 MiB unless it is the last. + public static readonly PART_SIZE = 8 * 1024 * 1024; + + public static readonly DEFAULT_MAX_MB = 100; + + private static readonly MAX_MB_CEILING = 1024; + + public static readonly PENDING_TTL_MS = 24 * 60 * 60 * 1000; + + // A file outlives its message by this much, so a message still on screen + // when its room's TTL runs out never shows a broken image. + public static readonly EXPIRY_GRACE_MS = 10 * 60 * 1000; + + public static readonly MAX_PENDING_PER_PLAYER = 20; + + public static readonly DEFAULT_DAILY_MB = 1024; + + private static readonly DAILY_MB_CEILING = 100 * 1024; + + // Per api pod: each part in flight holds a socket and a request to storage. + public static readonly MAX_UPLOADS_PER_PLAYER = 2; + + public static readonly MAX_UPLOADS_PER_PROCESS = 16; + + public static readonly MAX_PIXELS = 40_000_000; + + private static readonly SNIFF_BYTES = 16; + + // A jpeg's size sits after its exif and colour profile, which can run long. + private static readonly IMAGE_HEADER_BYTES = 1024 * 1024; + + public static readonly POSTER_MAX_BYTES = 2 * 1024 * 1024; + + private static readonly MAX_DIMENSION = 16384; + + private static readonly MAX_DURATION_MS = 24 * 60 * 60 * 1000; + + private static readonly MAX_NAME_LENGTH = 120; + + private static readonly SWEEP_BATCH = 500; + + public static readonly PREFIX = "chat-attachments"; + + private static readonly SCOPES = ["rooms", "direct"] as const; + + public static readonly TYPES: Record = { + "image/png": "image", + "image/jpeg": "image", + "image/webp": "image", + "image/gif": "image", + "video/mp4": "video", + "video/webm": "video", + "video/quicktime": "video", + }; + + private static readonly EXTENSIONS: Record = { + "image/png": "png", + "image/jpeg": "jpg", + "image/webp": "webp", + "image/gif": "gif", + "video/mp4": "mp4", + "video/webm": "webm", + "video/quicktime": "mov", + }; + + private static readonly POSTER_TYPES = [ + "image/webp", + "image/png", + "image/jpeg", + ]; + + // Match and match_team lines are relayed into the game server, which can + // only show text. + private static readonly ROOMS = new Set([ + ChatLobbyType.MatchMaking, + ChatLobbyType.Tournament, + ChatLobbyType.Organizer, + ChatLobbyType.Draft, + ChatLobbyType.Direct, + ]); + + private static readonly COLUMNS = `id::text AS id, + uploader_steam_id::text AS uploader_steam_id, room_type, room_id, + storage_prefix, file_name, mime_type, size::text AS size, width, + height, duration_ms, poster_mime_type, upload_id, + message_id::text AS message_id, deleted_at`; + + private readonly uploadsByPlayer = new Map(); + private uploadsInFlight = 0; + + constructor( + private readonly logger: Logger, + private readonly postgres: PostgresService, + private readonly s3: S3Service, + ) {} + + public static allowsAttachments(type: ChatLobbyType): boolean { + return ChatAttachmentsService.ROOMS.has(type); + } + + public static uploadRefusal( + upload: ChatAttachmentUpload, + maxBytes: number, + ): ChatErrorCode | null { + if ( + typeof upload.mime_type !== "string" || + !ChatAttachmentsService.TYPES[upload.mime_type] + ) { + return ChatErrorCode.UnsupportedType; + } + + if ( + typeof upload.size !== "number" || + !Number.isSafeInteger(upload.size) || + upload.size <= 0 + ) { + return ChatErrorCode.Invalid; + } + + if (upload.size > maxBytes) { + return ChatErrorCode.TooLarge; + } + + return null; + } + + public static maxFileBytesFrom(value: string | null | undefined): number { + const megabytes = Number.parseInt(value ?? "", 10); + + if (!Number.isFinite(megabytes) || megabytes <= 0) { + return ChatAttachmentsService.DEFAULT_MAX_MB * 1024 * 1024; + } + + return ( + Math.min(megabytes, ChatAttachmentsService.MAX_MB_CEILING) * 1024 * 1024 + ); + } + + public static dailyQuotaBytesFrom(value: string | null | undefined): number { + const megabytes = Number.parseInt(value ?? "", 10); + + if (!Number.isFinite(megabytes) || megabytes <= 0) { + return ChatAttachmentsService.DEFAULT_DAILY_MB * 1024 * 1024; + } + + return ( + Math.min(megabytes, ChatAttachmentsService.DAILY_MB_CEILING) * 1024 * 1024 + ); + } + + // Named for what the bytes are, so the extension a browser goes by can + // never disagree with the type the file is served as. + public static downloadName(name: string, mimeType: string): string { + const extension = ChatAttachmentsService.EXTENSIONS[mimeType] ?? "bin"; + const dot = name.lastIndexOf("."); + const base = (dot > 0 ? name.slice(0, dot) : name).trim() || "file"; + + return `${base}.${extension}`; + } + + public static imageSize( + header: Buffer, + mimeType: string, + ): { width: number; height: number } | null { + const size = ChatAttachmentsService.readImageSize(header, mimeType); + + if (!size || size.width <= 0 || size.height <= 0) { + return null; + } + + return size; + } + + private static readImageSize( + header: Buffer, + mimeType: string, + ): { width: number; height: number } | null { + const ascii = (start: number, end: number) => + header.subarray(start, end).toString("latin1"); + + switch (mimeType) { + case "image/png": + if (header.length < 24 || ascii(12, 16) !== "IHDR") { + return null; + } + + return { + width: header.readUInt32BE(16), + height: header.readUInt32BE(20), + }; + case "image/gif": + if (header.length < 10) { + return null; + } + + return { + width: header.readUInt16LE(6), + height: header.readUInt16LE(8), + }; + case "image/webp": + return ChatAttachmentsService.webpSize(header, ascii(12, 16)); + case "image/jpeg": + return ChatAttachmentsService.jpegSize(header); + default: + return null; + } + } + + private static webpSize( + header: Buffer, + chunk: string, + ): { width: number; height: number } | null { + if (chunk === "VP8X" && header.length >= 30) { + return { + width: header.readUIntLE(24, 3) + 1, + height: header.readUIntLE(27, 3) + 1, + }; + } + + if ( + chunk === "VP8 " && + header.length >= 30 && + header[23] === 0x9d && + header[24] === 0x01 && + header[25] === 0x2a + ) { + return { + width: header.readUInt16LE(26) & 0x3fff, + height: header.readUInt16LE(28) & 0x3fff, + }; + } + + if (chunk === "VP8L" && header.length >= 25 && header[20] === 0x2f) { + const bits = header.readUInt32LE(21); + + return { + width: (bits & 0x3fff) + 1, + height: ((bits >>> 14) & 0x3fff) + 1, + }; + } + + return null; + } + + // Each frame's image descriptor carries its own size, and a decoder sizes + // its canvas from the first one whatever the screen size at the top of the + // file says -- so a GIF whose first frame lies past `header` is refused. + public static gifFrames(header: Buffer): { + first: { width: number; height: number } | null; + largest: number; + } { + let first: { width: number; height: number } | null = null; + let largest = 0; + + if (header.length < 13) { + return { first, largest }; + } + + let offset = 13; + + if (header[10] & 0x80) { + offset += 3 * 2 ** ((header[10] & 0x07) + 1); + } + + const skipSubBlocks = (from: number) => { + let at = from; + + while (at < header.length && header[at] !== 0) { + at += header[at] + 1; + } + + return at + 1; + }; + + while (offset < header.length) { + const block = header[offset]; + + if (block === 0x3b) { + break; + } + + if (block === 0x21) { + offset = skipSubBlocks(offset + 2); + continue; + } + + if (block !== 0x2c || offset + 10 > header.length) { + break; + } + + const width = header.readUInt16LE(offset + 5); + const height = header.readUInt16LE(offset + 7); + + first ??= { width, height }; + largest = Math.max(largest, width * height); + + const packed = header[offset + 9]; + offset += 10; + + if (packed & 0x80) { + offset += 3 * 2 ** ((packed & 0x07) + 1); + } + + offset = skipSubBlocks(offset + 1); + } + + return { first, largest }; + } + + // A strong, quoted ETag from what storage reports, which files-sdk hands + // back unquoted. + public static etag(raw: string | undefined): string | undefined { + const bare = raw?.replace(/^W\//, "").replace(/"/g, ""); + + return bare ? `"${bare}"` : undefined; + } + + // If-None-Match compares weakly, so a W/ prefix still matches. + public static notModified( + header: string | undefined, + etag: string | undefined, + ): boolean { + if (!header || !etag) { + return false; + } + + return header + .split(",") + .map((value) => value.trim()) + .some((value) => value === "*" || value.replace(/^W\//, "") === etag); + } + + // Walks the segments to the first start-of-frame. DHT (C4), JPG (C8) and + // DAC (CC) share the C0-CF range without being frames. + private static jpegSize( + header: Buffer, + ): { width: number; height: number } | null { + let offset = 2; + + while (offset + 4 <= header.length) { + if (header[offset] !== 0xff) { + return null; + } + + const marker = header[offset + 1]; + + if (marker === 0xff) { + offset++; + continue; + } + + if (marker === 0x01 || (marker >= 0xd0 && marker <= 0xd8)) { + offset += 2; + continue; + } + + if (marker === 0xda || marker === 0xd9) { + return null; + } + + if ( + marker >= 0xc0 && + marker <= 0xcf && + ![0xc4, 0xc8, 0xcc].includes(marker) + ) { + if (offset + 9 > header.length) { + return null; + } + + return { + width: header.readUInt16BE(offset + 7), + height: header.readUInt16BE(offset + 5), + }; + } + + offset += 2 + header.readUInt16BE(offset + 2); + } + + return null; + } + + public static fileName(raw: unknown): string { + if (typeof raw !== "string") { + return "file"; + } + + const base = raw + .split(/[\\/]/) + .pop() + // eslint-disable-next-line no-control-regex + .replace(/[\u0000-\u001f\u007f"]/g, "") + .trim(); + + if (!base) { + return "file"; + } + + const max = ChatAttachmentsService.MAX_NAME_LENGTH; + + if (base.length <= max) { + return base; + } + + const dot = base.lastIndexOf("."); + const extension = dot > 0 && base.length - dot <= 10 ? base.slice(dot) : ""; + + return `${base.slice(0, max - extension.length)}${extension}`; + } + + public static sniff(header: Buffer): string | null { + const ascii = (start: number, end: number) => + header.subarray(start, end).toString("latin1"); + + if ( + header.length >= 8 && + header + .subarray(0, 8) + .equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])) + ) { + return "image/png"; + } + + if ( + header.length >= 3 && + header[0] === 0xff && + header[1] === 0xd8 && + header[2] === 0xff + ) { + return "image/jpeg"; + } + + if (["GIF87a", "GIF89a"].includes(ascii(0, 6))) { + return "image/gif"; + } + + if (ascii(0, 4) === "RIFF" && ascii(8, 12) === "WEBP") { + return "image/webp"; + } + + if ( + header.length >= 4 && + header.subarray(0, 4).equals(Buffer.from([0x1a, 0x45, 0xdf, 0xa3])) + ) { + return "video/webm"; + } + + if (ascii(4, 8) === "ftyp") { + return ascii(8, 12) === "qt " ? "video/quicktime" : "video/mp4"; + } + + // QuickTime files older than the ftyp box open straight on another atom. + if (["moov", "mdat", "wide", "free", "skip"].includes(ascii(4, 8))) { + return "video/quicktime"; + } + + return null; + } + + // A browser names a file's type from its extension, so a jpeg saved as .png + // is still an image -- what is served is what the bytes are, as long as it is + // the kind that was declared. + public static contentType(declared: string, header: Buffer): string | null { + const sniffed = ChatAttachmentsService.sniff(header); + + if ( + !sniffed || + ChatAttachmentsService.TYPES[sniffed] !== + ChatAttachmentsService.TYPES[declared] + ) { + return null; + } + + return sniffed; + } + + public static posterType(header: Buffer): string | null { + const sniffed = ChatAttachmentsService.sniff(header); + + return sniffed && ChatAttachmentsService.POSTER_TYPES.includes(sniffed) + ? sniffed + : null; + } + + public static partCount(size: number): number { + return Math.ceil(size / ChatAttachmentsService.PART_SIZE); + } + + public static partLength(size: number, part: number): number | null { + const parts = ChatAttachmentsService.partCount(size); + + if (!Number.isInteger(part) || part < 1 || part > parts) { + return null; + } + + if (part < parts) { + return ChatAttachmentsService.PART_SIZE; + } + + return size - (parts - 1) * ChatAttachmentsService.PART_SIZE; + } + + public static pendingExpiry(now: Date): Date { + return new Date(now.getTime() + ChatAttachmentsService.PENDING_TTL_MS); + } + + public static expiresOnSend( + type: ChatLobbyType, + ttlSeconds: number, + now: Date, + ): Date | null { + if (type === ChatLobbyType.Direct) { + return null; + } + + return new Date( + now.getTime() + + ttlSeconds * 1000 + + ChatAttachmentsService.EXPIRY_GRACE_MS, + ); + } + + // Filed by the day it was made, so the daily sweep can drop a whole day that + // nothing points at in one call. Direct messages keep theirs for as long as + // retention says, a room for a day or so -- apart, a lingering direct message + // never holds a day of room files back. + public static storagePrefix( + type: ChatLobbyType, + id: string, + createdAt: Date, + ): string { + const scope = type === ChatLobbyType.Direct ? "direct" : "rooms"; + const day = createdAt.toISOString().slice(0, 10); + + return `${ChatAttachmentsService.PREFIX}/${scope}/${day}/${id}/`; + } + + // A deleted message's files are evidence: gone from the room, its author + // included, but kept for staff until they expire. The organizers' room's + // evidence is closed to moderators, so its files are too. + public static async canView( + row: Pick< + ChatAttachmentRow, + "uploader_steam_id" | "message_id" | "room_type" | "deleted_at" + >, + viewer: ChatAttachmentViewer | undefined, + roomAccess: () => Promise, + ): Promise { + if (!viewer?.steam_id) { + return false; + } + + if (row.deleted_at) { + return isRoleAbove( + viewer.role, + row.room_type === ChatLobbyType.Organizer + ? "match_organizer" + : "moderator", + ); + } + + if (row.uploader_steam_id === String(viewer.steam_id)) { + return true; + } + + if (!row.message_id) { + return false; + } + + return await roomAccess(); + } + + public static parseRange( + header: string, + size: number, + ): { start: number; end: number } | null { + const match = /^bytes=(\d*)-(\d*)$/.exec(header.trim()); + + if (!match || (match[1] === "" && match[2] === "")) { + return null; + } + + let start: number; + let end: number; + + if (match[1] === "") { + const suffix = Number.parseInt(match[2], 10); + + if (suffix <= 0) { + return null; + } + + start = Math.max(0, size - suffix); + end = size - 1; + } else { + start = Number.parseInt(match[1], 10); + end = match[2] === "" ? size - 1 : Number.parseInt(match[2], 10); + } + + if (start >= size || end < start) { + return null; + } + + return { start, end: Math.min(end, size - 1) }; + } + + private static dimension(raw: unknown, max: number): number | null { + return typeof raw === "number" && + Number.isInteger(raw) && + raw > 0 && + raw <= max + ? raw + : null; + } + + public static descriptor(row: ChatAttachmentRow): ChatAttachment { + const kind = ChatAttachmentsService.TYPES[row.mime_type] ?? "image"; + + return { + id: row.id, + kind, + name: row.file_name, + mime_type: row.mime_type, + size: Number(row.size), + ...(row.width ? { width: row.width } : {}), + ...(row.height ? { height: row.height } : {}), + ...(row.duration_ms ? { duration_ms: row.duration_ms } : {}), + ...(row.poster_mime_type ? { poster: true } : {}), + }; + } + + public async maxFileBytes(): Promise { + const [row] = await this.postgres.query>( + `SELECT value FROM public.settings WHERE name = $1`, + [SystemSettingName.ChatAttachmentMaxMb], + ); + + return ChatAttachmentsService.maxFileBytesFrom(row?.value); + } + + public async dailyQuotaBytes(): Promise { + const [row] = await this.postgres.query>( + `SELECT value FROM public.settings WHERE name = $1`, + [SystemSettingName.ChatAttachmentDailyMb], + ); + + return ChatAttachmentsService.dailyQuotaBytesFrom(row?.value); + } + + // The row goes in before anything reaches storage, so there is never a file + // without something that will sweep it. + public async create( + steamId: string, + type: ChatLobbyType, + roomId: string, + upload: ChatAttachmentUpload, + ): Promise< + { id: string; part_size: number; parts: number } | { code: ChatErrorCode } + > { + const refusal = ChatAttachmentsService.uploadRefusal( + upload, + await this.maxFileBytes(), + ); + + if (refusal) { + return { code: refusal }; + } + + const size = upload.size as number; + const mimeType = upload.mime_type as string; + const id = randomUUID(); + const now = new Date(); + const prefix = ChatAttachmentsService.storagePrefix(type, id, now); + const dailyBytes = await this.dailyQuotaBytes(); + + // One player's uploads take turns here, so a burst of them cannot all + // count the same allowance before any of them is written. The usage + // ledger outlives the files, or uploading and removing the same file over + // and over would cost nothing. + const quotaRefusal = await this.postgres.transaction(async (client) => { + await client.query( + `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, + [`chat_attachments:${steamId}`], + ); + + const { + rows: [usage], + } = await client.query<{ pending: number; bytes: string }>( + `SELECT (SELECT count(*)::int + FROM public.chat_attachments + WHERE uploader_steam_id = $1::bigint + AND message_id IS NULL + AND expires_at > now()) AS pending, + (SELECT COALESCE(sum(bytes), 0)::text + FROM public.chat_attachment_usage + WHERE steam_id = $1::bigint + AND created_at > now() - interval '1 day') AS bytes`, + [steamId], + ); + + if (usage.pending >= ChatAttachmentsService.MAX_PENDING_PER_PLAYER) { + return ChatErrorCode.TooManyPending; + } + + if (Number(usage.bytes) + size > dailyBytes) { + return ChatErrorCode.QuotaExceeded; + } + + await client.query( + `INSERT INTO public.chat_attachments + (id, uploader_steam_id, room_type, room_id, storage_prefix, + file_name, mime_type, size, width, height, duration_ms, + expires_at, created_at) + VALUES ($1::uuid, $2::bigint, $3, $4, $5, $6, $7, $8::bigint, + $9::int, $10::int, $11::int, $12::timestamptz, + $13::timestamptz)`, + [ + id, + steamId, + type, + roomId, + prefix, + ChatAttachmentsService.fileName(upload.name), + mimeType, + size, + ChatAttachmentsService.dimension( + upload.width, + ChatAttachmentsService.MAX_DIMENSION, + ), + ChatAttachmentsService.dimension( + upload.height, + ChatAttachmentsService.MAX_DIMENSION, + ), + ChatAttachmentsService.TYPES[mimeType] === "video" + ? ChatAttachmentsService.dimension( + upload.duration_ms, + ChatAttachmentsService.MAX_DURATION_MS, + ) + : null, + ChatAttachmentsService.pendingExpiry(now).toISOString(), + now.toISOString(), + ], + ); + + await client.query( + `INSERT INTO public.chat_attachment_usage (steam_id, bytes) + VALUES ($1::bigint, $2::bigint)`, + [steamId, size], + ); + + return null; + }); + + if (quotaRefusal) { + return { code: quotaRefusal }; + } + + try { + const uploadId = await this.s3.createMultipartUpload(`${prefix}file`); + + await this.postgres.query( + `UPDATE public.chat_attachments SET upload_id = $2 WHERE id = $1::uuid`, + [id, uploadId], + ); + } catch (error) { + await this.postgres.query( + `DELETE FROM public.chat_attachments WHERE id = $1::uuid`, + [id], + ); + throw error; + } + + return { + id, + part_size: ChatAttachmentsService.PART_SIZE, + parts: ChatAttachmentsService.partCount(size), + }; + } + + private async uploading( + steamId: string, + id: string, + ): Promise { + const [row] = await this.postgres.query>( + `SELECT ${ChatAttachmentsService.COLUMNS} + FROM public.chat_attachments + WHERE id = $1::uuid + AND uploader_steam_id = $2::bigint + AND upload_id IS NOT NULL + AND message_id IS NULL + AND expires_at > now()`, + [id, steamId], + ); + + return row; + } + + private acquireUploadSlot(steamId: string): (() => void) | null { + const mine = this.uploadsByPlayer.get(steamId) ?? 0; + + if ( + mine >= ChatAttachmentsService.MAX_UPLOADS_PER_PLAYER || + this.uploadsInFlight >= ChatAttachmentsService.MAX_UPLOADS_PER_PROCESS + ) { + return null; + } + + this.uploadsByPlayer.set(steamId, mine + 1); + this.uploadsInFlight++; + + let released = false; + + return () => { + if (released) { + return; + } + + released = true; + this.uploadsInFlight--; + + const left = (this.uploadsByPlayer.get(steamId) ?? 1) - 1; + + if (left > 0) { + this.uploadsByPlayer.set(steamId, left); + return; + } + + this.uploadsByPlayer.delete(steamId); + }; + } + + private static async readHead( + stream: Readable, + bytes: number, + ): Promise<{ head: Buffer; rest: Readable }> { + const iterator = stream[Symbol.asyncIterator](); + const chunks: Buffer[] = []; + let length = 0; + let ended = false; + + while (length < bytes) { + const next = await iterator.next(); + + if (next.done) { + ended = true; + break; + } + + const chunk = Buffer.from(next.value); + chunks.push(chunk); + length += chunk.length; + } + + async function* replay() { + yield* chunks; + + if (ended) { + return; + } + + while (true) { + const next = await iterator.next(); + + if (next.done) { + return; + } + + yield next.value as Buffer; + } + } + + return { + head: Buffer.concat(chunks).subarray(0, bytes), + rest: Readable.from(replay(), { objectMode: false }), + }; + } + + // Through pipeline, so an error on either side is handled here: the SDK + // only pipes the body on, and a sender that drops mid-part would otherwise + // re-emit an 'error' nothing listens for, which takes the process down. The + // signal is what cancels the request to storage when that happens. + private static exactly( + source: Readable, + length: number, + ): { stream: Readable; failed: () => boolean; signal: AbortSignal } { + let seen = 0; + let failed = false; + const controller = new AbortController(); + + const counter = new Transform({ + transform(chunk: Buffer, _encoding, callback) { + seen += chunk.length; + + if (seen > length) { + callback(new Error("part is longer than it said")); + return; + } + + callback(null, chunk); + }, + flush(callback) { + if (seen !== length) { + callback(new Error("part is shorter than it said")); + return; + } + + callback(); + }, + }); + + pipeline(source, counter, (error) => { + if (error) { + failed = true; + controller.abort(error); + } + }); + + return { + stream: counter, + failed: () => failed, + signal: controller.signal, + }; + } + + // The first part is where the bytes say what the file really is, so that is + // where anything else is turned away -- before any of it reaches storage. + // The upload is authorized before a byte of the body is read, and the part + // is streamed through rather than held. + public async uploadPart( + steamId: string, + id: string, + part: number, + body: Readable | Buffer, + declaredLength?: number, + ): Promise { + const stream = Buffer.isBuffer(body) ? Readable.from([body]) : body; + const length = + declaredLength ?? (Buffer.isBuffer(body) ? body.length : Number.NaN); + + const release = this.acquireUploadSlot(steamId); + + if (!release) { + return ChatErrorCode.RateLimited; + } + + try { + const row = await this.uploading(steamId, id); + + if (!row) { + return ChatErrorCode.NotFound; + } + + const expected = ChatAttachmentsService.partLength( + Number(row.size), + part, + ); + + if (expected === null || length !== expected) { + return ChatErrorCode.Invalid; + } + + let source = stream; + + if (part === 1) { + let read: { head: Buffer; rest: Readable }; + + try { + read = await ChatAttachmentsService.readHead( + stream, + Math.min( + expected, + ChatAttachmentsService.TYPES[row.mime_type] === "image" + ? ChatAttachmentsService.IMAGE_HEADER_BYTES + : ChatAttachmentsService.SNIFF_BYTES, + ), + ); + } catch { + return ChatErrorCode.Invalid; + } + + const { head, rest } = read; + + const refusal = await this.checkFirstPart(row, head); + + if (refusal) { + return refusal; + } + + source = rest; + } + + const checked = ChatAttachmentsService.exactly(source, expected); + + try { + await this.s3.uploadPart( + `${row.storage_prefix}file`, + row.upload_id, + part, + checked.stream, + expected, + checked.signal, + ); + } catch (error) { + if (checked.failed()) { + return ChatErrorCode.Invalid; + } + + this.logger.warn( + `unable to store part ${part} of chat attachment ${id}`, + error, + ); + return ChatErrorCode.Unavailable; + } + + return null; + } finally { + release(); + } + } + + // An image's real size comes from its own header: what the browser said is + // only a guess, and a few kilobytes can claim to decode to gigapixels. + private async checkFirstPart( + row: ChatAttachmentRow, + head: Buffer, + ): Promise { + const contentType = ChatAttachmentsService.contentType( + row.mime_type, + head.subarray(0, ChatAttachmentsService.SNIFF_BYTES), + ); + + if (!contentType) { + await this.expire([row.id]); + return ChatErrorCode.UnsupportedType; + } + + let width = row.width; + let height = row.height; + + if (ChatAttachmentsService.TYPES[contentType] === "image") { + const size = ChatAttachmentsService.imageSize(head, contentType); + + if ( + !size || + (contentType === "image/gif" && + !ChatAttachmentsService.gifFrames(head).first) + ) { + await this.expire([row.id]); + return ChatErrorCode.UnsupportedType; + } + + if ( + size.width * size.height > ChatAttachmentsService.MAX_PIXELS || + (contentType === "image/gif" && + ChatAttachmentsService.gifFrames(head).largest > + ChatAttachmentsService.MAX_PIXELS) + ) { + await this.expire([row.id]); + return ChatErrorCode.TooLarge; + } + + ({ width, height } = size); + } + + await this.postgres.query( + `UPDATE public.chat_attachments + SET mime_type = $2, width = $3, height = $4 + WHERE id = $1::uuid`, + [row.id, contentType, width, height], + ); + + return null; + } + + public async complete( + steamId: string, + id: string, + ): Promise { + const row = await this.uploading(steamId, id); + + if (!row) { + return { code: ChatErrorCode.NotFound }; + } + + const key = `${row.storage_prefix}file`; + + try { + await this.s3.completeMultipartUpload(key, row.upload_id); + } catch (error) { + this.logger.warn(`unable to complete chat attachment ${id}`, error); + return { code: ChatErrorCode.Invalid }; + } + + const stored = await this.s3.stat(key); + + if (stored.size !== Number(row.size)) { + await this.postgres.query( + `UPDATE public.chat_attachments + SET upload_id = NULL, expires_at = now() + WHERE id = $1::uuid`, + [row.id], + ); + await this.remove([{ ...row, upload_id: null }]); + return { code: ChatErrorCode.Invalid }; + } + + const [completed] = await this.postgres.query>( + `UPDATE public.chat_attachments + SET upload_id = NULL, uploaded_at = now() + WHERE id = $1::uuid + RETURNING ${ChatAttachmentsService.COLUMNS}`, + [row.id], + ); + + return ChatAttachmentsService.descriptor(completed); + } + + // Read whole only once the upload is known to be the player's, and only up + // to the poster cap. + public async setPoster( + steamId: string, + id: string, + body: Readable | Buffer, + declaredLength?: number, + ): Promise { + const length = + declaredLength ?? (Buffer.isBuffer(body) ? body.length : Number.NaN); + + if (!(length > 0) || length > ChatAttachmentsService.POSTER_MAX_BYTES) { + return ChatErrorCode.TooLarge; + } + + const release = this.acquireUploadSlot(steamId); + + if (!release) { + return ChatErrorCode.RateLimited; + } + + try { + const [row] = await this.postgres.query>( + `SELECT ${ChatAttachmentsService.COLUMNS} + FROM public.chat_attachments + WHERE id = $1::uuid + AND uploader_steam_id = $2::bigint + AND message_id IS NULL + AND expires_at > now()`, + [id, steamId], + ); + + if (!row) { + return ChatErrorCode.NotFound; + } + + if (ChatAttachmentsService.TYPES[row.mime_type] !== "video") { + return ChatErrorCode.Invalid; + } + + const checked = ChatAttachmentsService.exactly( + Buffer.isBuffer(body) ? Readable.from([body]) : body, + length, + ); + const chunks: Buffer[] = []; + + try { + for await (const chunk of checked.stream) { + chunks.push(chunk as Buffer); + } + } catch { + return ChatErrorCode.Invalid; + } + + const poster = Buffer.concat(chunks); + const posterType = ChatAttachmentsService.posterType( + poster.subarray(0, ChatAttachmentsService.SNIFF_BYTES), + ); + + if (!posterType) { + return ChatErrorCode.UnsupportedType; + } + + await this.s3.put(`${row.storage_prefix}poster`, poster, posterType); + + await this.postgres.query( + `UPDATE public.chat_attachments + SET poster_mime_type = $2 + WHERE id = $1::uuid`, + [row.id, posterType], + ); + + return null; + } finally { + release(); + } + } + + // Taken out of the tray before it was ever sent. + public async discard(steamId: string, id: string): Promise { + const rows = await this.postgres.query>( + `UPDATE public.chat_attachments + SET expires_at = now() + WHERE id = $1::uuid + AND uploader_steam_id = $2::bigint + AND message_id IS NULL + RETURNING id::text AS id, storage_prefix, upload_id`, + [id, steamId], + ); + + if (rows.length === 0) { + return false; + } + + await this.remove(rows); + + return true; + } + + // All of them or none: a message is never sent missing a file it was + // composed with. FOR UPDATE makes a second send of the same file wait for + // the first, then find it taken. + public async claim( + ids: string[], + claim: ChatAttachmentClaim, + client?: PoolClient, + ): Promise { + if (ids.length === 0) { + return []; + } + + const sql = `WITH eligible AS ( + SELECT id AS eligible_id + FROM public.chat_attachments + WHERE id = ANY($1::uuid[]) + AND uploader_steam_id = $2::bigint + AND room_type = $3 + AND room_id = $4 + AND message_id IS NULL + AND deleted_at IS NULL + AND uploaded_at IS NOT NULL + AND expires_at > now() + FOR UPDATE + ) + UPDATE public.chat_attachments + SET message_id = $5::uuid, + sent_at = now(), + expires_at = $6::timestamptz + FROM eligible + WHERE id = eligible.eligible_id + AND (SELECT count(*) FROM eligible) = cardinality($1::uuid[]) + RETURNING ${ChatAttachmentsService.COLUMNS}`; + + const bindings = [ + ids, + claim.steamId, + claim.type, + claim.roomId, + claim.messageId, + claim.expiresAt?.toISOString() ?? null, + ]; + + const rows: ChatAttachmentRow[] = client + ? (await client.query(sql, bindings)).rows + : await this.postgres.query>(sql, bindings); + + if (rows.length !== ids.length) { + return null; + } + + return ids.map((id) => + ChatAttachmentsService.descriptor(rows.find((row) => row.id === id)), + ); + } + + // Every one of the files already went out with a message from this sender, + // in this room. + public async sentBy( + ids: string[], + claim: Pick, + ): Promise { + const [row] = await this.postgres.query>( + `SELECT count(*)::int AS sent + FROM public.chat_attachments + WHERE id = ANY($1::uuid[]) + AND uploader_steam_id = $2::bigint + AND room_type = $3 + AND room_id = $4 + AND message_id IS NOT NULL`, + [ids, claim.steamId, claim.type, claim.roomId], + ); + + return (row?.sent ?? 0) === ids.length; + } + + // A group room's deleted message keeps its files as evidence: hidden from + // the room, kept for staff, and swept at the expiry they already had. + public async markDeleted( + type: ChatLobbyType, + roomId: string, + messageId: string, + ): Promise { + await this.postgres.query( + `UPDATE public.chat_attachments + SET deleted_at = now() + WHERE message_id = $1::uuid + AND room_type = $2 + AND room_id = $3 + AND deleted_at IS NULL`, + [messageId, type, roomId], + ); + } + + // A direct message is not moderated, so deleting one takes its files with + // it at once. A failed sweep is left to the next removeExpired. + public async expireMessage( + type: ChatLobbyType, + roomId: string, + messageId: string, + ): Promise { + const rows = await this.postgres.query>( + `UPDATE public.chat_attachments + SET expires_at = now() + WHERE message_id = $1::uuid + AND room_type = $2 + AND room_id = $3 + RETURNING id::text AS id, storage_prefix, upload_id`, + [messageId, type, roomId], + ); + + await this.remove(rows); + } + + // A draft's chat carries on in its match room, where the moved messages take + // that room's TTL from now. Their files follow, and are judged by who can + // open the match. + public async moveRoom( + fromType: ChatLobbyType, + fromId: string, + toType: ChatLobbyType, + toId: string, + expiresAt: Date | null, + ): Promise { + await this.postgres.query( + `UPDATE public.chat_attachments + SET room_type = $3, + room_id = $4, + expires_at = GREATEST(expires_at, $5::timestamptz) + WHERE room_type = $1 + AND room_id = $2 + AND message_id IS NOT NULL + AND deleted_at IS NULL + AND expires_at > now()`, + [fromType, fromId, toType, toId, expiresAt?.toISOString() ?? null], + ); + } + + public async find(id: string): Promise { + const [row] = await this.postgres.query>( + `SELECT ${ChatAttachmentsService.COLUMNS} + FROM public.chat_attachments + WHERE id = $1::uuid + AND uploaded_at IS NOT NULL + AND (expires_at IS NULL OR expires_at > now())`, + [id], + ); + + return row; + } + + public async removeExpired(): Promise { + await this.postgres.query( + `DELETE FROM public.chat_attachment_usage + WHERE created_at < now() - interval '1 day'`, + ); + + const rows = await this.postgres.query>( + `SELECT id::text AS id, storage_prefix, upload_id + FROM public.chat_attachments + WHERE expires_at <= now() + ORDER BY expires_at + LIMIT ${ChatAttachmentsService.SWEEP_BATCH}`, + ); + + return await this.remove(rows); + } + + private async expire(ids: string[]) { + await this.postgres.query( + `UPDATE public.chat_attachments + SET expires_at = now() + WHERE id = ANY($1::uuid[])`, + [ids], + ); + } + + // Storage first, then the row: the row is the only thing that knows the file + // is there, so a sweep that fails has to leave it for the next run. + private async remove(rows: Removable[]): Promise { + let removed = 0; + + for (const row of rows) { + if (row.upload_id) { + try { + await this.s3.abortMultipartUpload( + `${row.storage_prefix}file`, + row.upload_id, + ); + } catch (error) { + if (!ChatAttachmentsService.isNoSuchUpload(error)) { + this.logger.warn( + `unable to abort the upload of chat attachment ${row.id}, will retry`, + error, + ); + continue; + } + } + + await this.postgres.query( + `UPDATE public.chat_attachments SET upload_id = NULL WHERE id = $1::uuid`, + [row.id], + ); + } + + try { + await this.s3.removePrefixStrictly(row.storage_prefix); + } catch (error) { + this.logger.warn( + `unable to remove chat attachment ${row.id}, will retry`, + error, + ); + continue; + } + + await this.postgres.query( + `DELETE FROM public.chat_attachments + WHERE id = $1::uuid + AND expires_at <= now() + AND upload_id IS NULL`, + [row.id], + ); + + removed++; + } + + return removed; + } + + private static isNoSuchUpload(error: unknown): boolean { + const failure = error as { + name?: string; + Code?: string; + $metadata?: { httpStatusCode?: number }; + }; + + return ( + failure?.name === "NoSuchUpload" || + failure?.Code === "NoSuchUpload" || + failure?.$metadata?.httpStatusCode === 404 + ); + } + + // Anything left in storage that no row points at: a sweep whose row went + // some other way, or a crash between the two. A day nothing points at goes in + // one call. Today and yesterday are left alone while uploads may still land. + public async sweepOrphans(now: Date = new Date()): Promise { + const cutoff = new Date(now.getTime() - 24 * 60 * 60 * 1000) + .toISOString() + .slice(0, 10); + + let removed = 0; + + for (const scope of ChatAttachmentsService.SCOPES) { + const root = `${ChatAttachmentsService.PREFIX}/${scope}/`; + + for (const day of await this.s3.listPrefixes(root)) { + const date = day.slice(root.length, -1); + + if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || date >= cutoff) { + continue; + } + + try { + removed += await this.sweepDay(day); + } catch (error) { + this.logger.warn( + `unable to sweep chat attachments under ${day}, will retry`, + error, + ); + } + } + } + + if (removed > 0) { + this.logger.log(`swept ${removed} orphaned chat attachment object(s)`); + } + + return removed; + } + + private async sweepDay(day: string): Promise { + const rows = await this.postgres.query>( + `SELECT id::text AS id + FROM public.chat_attachments + WHERE storage_prefix LIKE $1`, + [`${day}%`], + ); + + if (rows.length === 0) { + return await this.s3.removePrefixStrictly(day); + } + + const known = new Set(rows.map(({ id }) => id)); + const strays = new Set(); + + for await (const object of this.s3.listStream(day)) { + const id = object.name.slice(day.length).split("/")[0]; + + if (id && !known.has(id)) { + strays.add(id); + } + } + + let removed = 0; + + for (const id of strays) { + removed += await this.s3.removePrefixStrictly(`${day}${id}/`); + } + + return removed; + } + + public async stream( + key: string, + contentType: string, + fileName: string, + request: Request, + response: Response, + ): Promise { + let size: number; + let etag: string | undefined; + + try { + const stored = await this.s3.stat(key); + size = stored.size; + etag = ChatAttachmentsService.etag(stored.etag); + } catch (error) { + const { code, name } = (error ?? {}) as { code?: string; name?: string }; + + if (code === "NotFound" || name === "NotFound") { + response.status(404).end(); + return; + } + + throw error; + } + + response.setHeader("Content-Type", contentType); + response.setHeader("Accept-Ranges", "bytes"); + response.setHeader("X-Content-Type-Options", "nosniff"); + response.setHeader("Content-Security-Policy", "sandbox"); + response.setHeader( + "Content-Disposition", + `inline; filename*=UTF-8''${encodeURIComponent(fileName)}`, + ); + // Revalidated on every view: who may open a room changes, and a deleted + // message's files must stop showing at once, not when a cache lets go. + response.setHeader("Cache-Control", "private, no-cache"); + + if (etag) { + response.setHeader("ETag", etag); + + if ( + ChatAttachmentsService.notModified( + request.headers["if-none-match"], + etag, + ) + ) { + response.status(304).end(); + return; + } + } + + const header = request.headers.range; + const range = header + ? ChatAttachmentsService.parseRange(header, size) + : null; + + if (header && !range) { + response.setHeader("Content-Range", `bytes */${size}`); + response.status(416).end(); + return; + } + + const stream = range + ? await this.s3.getPartial(key, range.start, range.end - range.start + 1) + : await this.s3.get(key); + + if (range) { + response.status(206); + response.setHeader( + "Content-Range", + `bytes ${range.start}-${range.end}/${size}`, + ); + response.setHeader("Content-Length", String(range.end - range.start + 1)); + } else { + response.status(200); + response.setHeader("Content-Length", String(size)); + } + + response.on("close", () => { + stream.destroy(); + }); + + stream.on("error", (error) => { + this.logger.warn(`unable to stream ${key}`, error); + response.destroy(); + }); + + stream.pipe(response); + } +} diff --git a/src/chat/chat-gifs.service.spec.ts b/src/chat/chat-gifs.service.spec.ts new file mode 100644 index 00000000..c2b38d94 --- /dev/null +++ b/src/chat/chat-gifs.service.spec.ts @@ -0,0 +1,330 @@ +import { ChatGifsService } from "./chat-gifs.service"; + +describe("ChatGifsService", () => { + const KEY = "giphy-secret-key"; + + let key: string | null; + let hourly: string | null; + let cache: Map; + let counters: Map; + + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() }; + + const postgres = { + query: jest.fn(async (_sql: string, [name]: string[]) => { + const value = + name === "giphy_api_key" + ? key + : name === "giphy_hourly_limit" + ? hourly + : null; + + return value === null ? [] : [{ value }]; + }), + }; + + const redis = { + get: jest.fn(async (name: string) => cache.get(name) ?? null), + set: jest.fn( + async ( + name: string, + value: string, + _mode?: string, + _seconds?: number, + ) => { + cache.set(name, value); + return "OK"; + }, + ), + eval: jest.fn(async (_script: string, _keys: number, name: string) => { + const count = (counters.get(name) ?? 0) + 1; + counters.set(name, count); + return count; + }), + }; + + const giphy = (gifs: Array>, offset = 0) => ({ + ok: true, + status: 200, + json: async () => ({ + data: gifs, + pagination: { total_count: 100, count: gifs.length, offset }, + }), + }); + + const gif = (id: string) => ({ + id, + title: `${id} title`, + images: { + original: { + url: `https://media0.giphy.com/${id}.gif`, + width: "480", + height: "270", + }, + fixed_width: { + url: `https://media0.giphy.com/${id}-200.gif`, + width: "200", + height: "113", + }, + }, + }); + + let fetchMock: jest.SpyInstance; + let service: ChatGifsService; + + beforeEach(() => { + jest.clearAllMocks(); + key = KEY; + cache = new Map(); + hourly = null; + counters = new Map(); + fetchMock = jest + .spyOn(global, "fetch") + .mockResolvedValue(giphy([gif("abc123")]) as any); + service = new ChatGifsService( + logger as any, + postgres as any, + { getConnection: () => redis } as any, + ); + }); + + afterEach(() => { + fetchMock.mockRestore(); + }); + + const requested = () => new URL(fetchMock.mock.calls.at(-1)[0] as string); + + describe("a GIF in a message", () => { + it("keeps the GIPHY id and the size to lay it out at", () => { + expect( + ChatGifsService.gif({ id: "abc123", width: 480, height: 270 }), + ).toEqual({ id: "abc123", width: 480, height: 270 }); + }); + + it("drops anything else the client sent along", () => { + expect( + ChatGifsService.gif({ + id: "abc123", + width: 480, + height: 270, + url: "https://evil.example/pixel.gif", + }), + ).toEqual({ id: "abc123", width: 480, height: 270 }); + }); + + it.each([ + [ + "a url for an id", + { id: "https://evil.example/x.gif", width: 1, height: 1 }, + ], + ["a path in the id", { id: "../abc", width: 1, height: 1 }], + ["no id", { width: 1, height: 1 }], + ["a fractional width", { id: "abc", width: 1.5, height: 1 }], + ["no height", { id: "abc", width: 10 }], + ["a zero width", { id: "abc", width: 0, height: 1 }], + ["an absurd size", { id: "abc", width: 100000, height: 1 }], + ["a string", "abc123"], + ["null", null], + ])("refuses %s", (_, raw) => { + expect(ChatGifsService.gif(raw)).toBeNull(); + }); + }); + + describe("search", () => { + it("is off when no key is set", async () => { + key = null; + + await expect(service.enabled()).resolves.toBe(false); + await expect(service.search("1", "gg", 0)).resolves.toBe("disabled"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + // The settings page reads "key set" back straight after saving. + it("sees a key the moment it is added or removed", async () => { + key = null; + await expect(service.enabled()).resolves.toBe(false); + + key = KEY; + await expect(service.enabled()).resolves.toBe(true); + await expect(service.search("1", "gg", 0)).resolves.not.toBe("disabled"); + + key = null; + await expect(service.enabled()).resolves.toBe(false); + }); + + it("applies a new hourly allowance at once", async () => { + await service.search("1", "first", 0); + + hourly = "1"; + + await expect(service.search("2", "second", 0)).resolves.toBe("busy"); + }); + + it("searches GIPHY with the operator's key and no rating filter", async () => { + await service.search("1", " clutch ", 0); + + const url = requested(); + expect(url.origin + url.pathname).toBe( + "https://api.giphy.com/v1/gifs/search", + ); + expect(url.searchParams.get("api_key")).toBe(KEY); + expect(url.searchParams.get("q")).toBe("clutch"); + expect(url.searchParams.has("rating")).toBe(false); + }); + + it("shows what is trending until something is typed", async () => { + await service.search("1", " ", 0); + + const url = requested(); + expect(url.pathname).toBe("/v1/gifs/trending"); + expect(url.searchParams.has("q")).toBe(false); + expect(url.searchParams.has("rating")).toBe(false); + }); + + it("hands back ids and sizes, never a URL carrying the key", async () => { + const page = await service.search("1", "gg", 0); + + expect(page).toEqual({ + results: [ + { id: "abc123", title: "abc123 title", width: 480, height: 270 }, + ], + next: 1, + }); + expect(JSON.stringify(page)).not.toContain(KEY); + }); + + it("pages on from where the last one ended", async () => { + fetchMock.mockResolvedValue(giphy([gif("x1"), gif("x2")], 24) as any); + + const page = await service.search("1", "gg", 24); + + expect(requested().searchParams.get("offset")).toBe("24"); + expect(page).toMatchObject({ next: 26 }); + }); + + it("stops paging at the end of the results", async () => { + fetchMock.mockResolvedValue({ + ok: true, + status: 200, + json: async () => ({ + data: [gif("last")], + pagination: { total_count: 25, count: 1, offset: 24 }, + }), + } as any); + + await expect(service.search("1", "gg", 24)).resolves.toMatchObject({ + next: null, + }); + }); + + it("skips a result it could not lay out", async () => { + fetchMock.mockResolvedValue( + giphy([gif("good"), { id: "bad", images: {} }]) as any, + ); + + const page = await service.search("1", "gg", 0); + + expect(page).toMatchObject({ + results: [expect.objectContaining({ id: "good" })], + }); + }); + + it("answers a repeat search from the cache", async () => { + await service.search("1", "Clutch", 0); + await service.search("2", "clutch", 0); + + expect(fetchMock).toHaveBeenCalledTimes(1); + }); + + // The key is a shared quota, so one player cannot spend it for everyone. + it("limits how often one player can search", async () => { + counters.set("chat:gifs-rate:1", ChatGifsService.RATE_LIMIT); + + await expect(service.search("1", "gg", 0)).resolves.toBe("rate_limited"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("says GIPHY is unavailable rather than throwing", async () => { + fetchMock.mockResolvedValue({ ok: false, status: 429 } as any); + + await expect(service.search("1", "gg", 0)).resolves.toBe("unavailable"); + + fetchMock.mockRejectedValue(new Error("socket hang up")); + + await expect(service.search("1", "rage", 0)).resolves.toBe("unavailable"); + }); + + it.each([-1, 1.5, Number.NaN, 5000])( + "starts from the top for an offset of %p", + async (offset) => { + await service.search("1", "gg", offset); + + expect(requested().searchParams.get("offset")).toBe("0"); + }, + ); + + // One key serves the whole panel, and GIPHY's beta keys allow 100 calls an + // hour between everyone. + it("stops calling GIPHY once the panel's hourly allowance is spent", async () => { + for (let i = 0; i < ChatGifsService.DEFAULT_HOURLY_LIMIT; i++) { + await service.search(String(i), `term ${i}`, 0); + } + + fetchMock.mockClear(); + + await expect(service.search("99", "one more", 0)).resolves.toBe("busy"); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it("stays under GIPHY's beta limit unless the operator says otherwise", async () => { + expect(ChatGifsService.DEFAULT_HOURLY_LIMIT).toBeLessThan(100); + + hourly = "3"; + service = new ChatGifsService( + logger as any, + postgres as any, + { getConnection: () => redis } as any, + ); + + for (let i = 0; i < 3; i++) { + await service.search(String(i), `term ${i}`, 0); + } + + await expect(service.search("9", "fourth", 0)).resolves.toBe("busy"); + }); + + it("still answers from the cache when the allowance is spent", async () => { + await service.search("1", "gg", 0); + hourly = "0"; + service = new ChatGifsService( + logger as any, + postgres as any, + { getConnection: () => redis } as any, + ); + + await expect(service.search("2", "gg", 0)).resolves.toMatchObject({ + results: [expect.objectContaining({ id: "abc123" })], + }); + }); + + it("keeps what is trending longer than a search", async () => { + await service.search("1", "", 0); + await service.search("1", "gg", 0); + + const ttls = Object.fromEntries( + redis.set.mock.calls.map(([name, , , seconds]) => [name, seconds]), + ); + + expect(ttls["chat:gifs::0"]).toBe(60 * 60); + expect(ttls["chat:gifs:gg:0"]).toBe(10 * 60); + }); + + it("caps what can be searched for", async () => { + await service.search("1", "a".repeat(500), 0); + + expect(requested().searchParams.get("q")).toHaveLength( + ChatGifsService.MAX_QUERY_LENGTH, + ); + }); + }); +}); diff --git a/src/chat/chat-gifs.service.ts b/src/chat/chat-gifs.service.ts new file mode 100644 index 00000000..313c055d --- /dev/null +++ b/src/chat/chat-gifs.service.ts @@ -0,0 +1,280 @@ +import { Injectable, Logger } from "@nestjs/common"; +import Redis from "ioredis"; +import { PostgresService } from "../postgres/postgres.service"; +import { RedisManagerService } from "../redis/redis-manager/redis-manager.service"; +import { SystemSettingName } from "../system/enums/SystemSettingName"; +import { ChatGif } from "./types/ChatGif"; + +export interface ChatGifResult { + id: string; + title: string; + width: number; + height: number; +} + +export interface ChatGifPage { + results: ChatGifResult[]; + next: number | null; +} + +type GiphyImage = { width?: string; height?: string }; + +type GiphyGif = { + id?: string; + title?: string; + images?: { original?: GiphyImage; fixed_width?: GiphyImage }; +}; + +// Searches go through here so the operator's key never reaches a browser. Only +// GIPHY's answer is cached, for a few minutes -- the GIFs themselves are always +// loaded from GIPHY, as its terms ask. +@Injectable() +export class ChatGifsService { + public static readonly PAGE_SIZE = 24; + + public static readonly MAX_QUERY_LENGTH = 50; + + // The key is a quota shared by everyone on the panel. + public static readonly RATE_LIMIT = 30; + + private static readonly RATE_WINDOW_MS = 60_000; + + private static readonly CACHE_TTL_SECONDS = 600; + + private static readonly TRENDING_CACHE_TTL_SECONDS = 60 * 60; + + // GIPHY's beta keys allow 100 calls an hour, between everyone on the panel. + public static readonly DEFAULT_HOURLY_LIMIT = 90; + + // GIPHY refuses an offset past this. + private static readonly MAX_OFFSET = 4999; + + private static readonly MAX_DIMENSION = 4096; + + private static readonly ENDPOINT = "https://api.giphy.com/v1/gifs"; + + private static readonly ID = /^[A-Za-z0-9]{1,64}$/; + + private static readonly RATE_SCRIPT = ` + local count = redis.call('INCR', KEYS[1]) + if count == 1 then + redis.call('PEXPIRE', KEYS[1], ARGV[1]) + end + return count + `; + + private redis: Redis; + + constructor( + private readonly logger: Logger, + private readonly postgres: PostgresService, + private readonly redisManager: RedisManagerService, + ) { + this.redis = this.redisManager.getConnection(); + } + + public static gif(raw: unknown): ChatGif | null { + if (typeof raw !== "object" || raw === null) { + return null; + } + + const { id, width, height } = raw as Record; + + if (typeof id !== "string" || !ChatGifsService.ID.test(id)) { + return null; + } + + const size = (value: unknown) => + typeof value === "number" && + Number.isInteger(value) && + value > 0 && + value <= ChatGifsService.MAX_DIMENSION; + + if (!size(width) || !size(height)) { + return null; + } + + return { id, width: width as number, height: height as number }; + } + + public async enabled(): Promise { + return !!(await this.apiKey()); + } + + public async search( + steamId: string, + query: string, + offset: number, + ): Promise< + ChatGifPage | "disabled" | "rate_limited" | "busy" | "unavailable" + > { + const key = await this.apiKey(); + + if (!key) { + return "disabled"; + } + + const term = (query ?? "") + .trim() + .slice(0, ChatGifsService.MAX_QUERY_LENGTH); + const start = + Number.isInteger(offset) && + offset >= 0 && + offset <= ChatGifsService.MAX_OFFSET + ? offset + : 0; + + const cacheKey = `chat:gifs:${term.toLowerCase()}:${start}`; + const cached = await this.redis.get(cacheKey); + + if (cached !== null) { + return JSON.parse(cached) as ChatGifPage; + } + + const count = await this.redis.eval( + ChatGifsService.RATE_SCRIPT, + 1, + `chat:gifs-rate:${steamId}`, + ChatGifsService.RATE_WINDOW_MS, + ); + + if (Number(count) > ChatGifsService.RATE_LIMIT) { + return "rate_limited"; + } + + if (!(await this.withinHourlyLimit())) { + return "busy"; + } + + // No `rating`: the operator asked for GIPHY unfiltered, and leaving it out + // is how GIPHY's API says "every rating". + const url = new URL( + `${ChatGifsService.ENDPOINT}/${term ? "search" : "trending"}`, + ); + url.searchParams.set("api_key", key); + url.searchParams.set("limit", String(ChatGifsService.PAGE_SIZE)); + url.searchParams.set("offset", String(start)); + if (term) { + url.searchParams.set("q", term); + } + + let body: { + data?: GiphyGif[]; + pagination?: { total_count?: number; count?: number; offset?: number }; + }; + + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(10_000), + }); + + if (!response.ok) { + this.logger.warn(`[giphy] search answered ${response.status}`); + return "unavailable"; + } + + body = await response.json(); + } catch (error) { + this.logger.warn(`[giphy] search failed: ${(error as Error)?.message}`); + return "unavailable"; + } + + const page: ChatGifPage = { + results: (body.data ?? []) + .map((gif) => ChatGifsService.toResult(gif)) + .filter((result): result is ChatGifResult => result !== null), + next: ChatGifsService.nextOffset(start, body.pagination), + }; + + await this.redis.set( + cacheKey, + JSON.stringify(page), + "EX", + term + ? ChatGifsService.CACHE_TTL_SECONDS + : ChatGifsService.TRENDING_CACHE_TTL_SECONDS, + ); + + return page; + } + + private static toResult(gif: GiphyGif): ChatGifResult | null { + const image = gif.images?.original ?? gif.images?.fixed_width; + const width = Number(image?.width); + const height = Number(image?.height); + + if ( + typeof gif.id !== "string" || + !ChatGifsService.ID.test(gif.id) || + !(width > 0) || + !(height > 0) + ) { + return null; + } + + return { + id: gif.id, + title: typeof gif.title === "string" ? gif.title : "", + width: Math.min(Math.round(width), ChatGifsService.MAX_DIMENSION), + height: Math.min(Math.round(height), ChatGifsService.MAX_DIMENSION), + }; + } + + private static nextOffset( + start: number, + pagination?: { total_count?: number; count?: number }, + ): number | null { + const count = pagination?.count ?? 0; + const next = start + count; + + if ( + count === 0 || + next > ChatGifsService.MAX_OFFSET || + (typeof pagination?.total_count === "number" && + next >= pagination.total_count) + ) { + return null; + } + + return next; + } + + private async withinHourlyLimit(): Promise { + const hour = new Date().toISOString().slice(0, 13); + + const count = await this.redis.eval( + ChatGifsService.RATE_SCRIPT, + 1, + `chat:gifs-hourly:${hour}`, + 2 * 60 * 60 * 1000, + ); + + return Number(count) <= (await this.hourlyCalls()); + } + + private async hourlyCalls(): Promise { + const [row] = await this.postgres.query>( + `SELECT value FROM public.settings WHERE name = $1`, + [SystemSettingName.GiphyHourlyLimit], + ); + + const parsed = Number.parseInt(row?.value ?? "", 10); + return Number.isInteger(parsed) && parsed >= 0 + ? parsed + : ChatGifsService.DEFAULT_HOURLY_LIMIT; + } + + // Read straight from postgres, like PruneDirectMessages: SystemModule + // imports ChatModule, so going through SystemService closes a module cycle. + // Never cached: an api pod holding the old key would answer the settings + // page with the state from before it saved, and the lookup is one row by + // primary key next to a call out to GIPHY. + private async apiKey(): Promise { + const [row] = await this.postgres.query>( + `SELECT value FROM public.settings WHERE name = $1`, + [SystemSettingName.GiphyApiKey], + ); + + return row?.value?.trim() || null; + } +} diff --git a/src/chat/chat-media.controller.spec.ts b/src/chat/chat-media.controller.spec.ts new file mode 100644 index 00000000..703aa9d2 --- /dev/null +++ b/src/chat/chat-media.controller.spec.ts @@ -0,0 +1,379 @@ +import { HttpException } from "@nestjs/common"; +import { Readable } from "stream"; +import { ChatMediaController } from "./chat-media.controller"; +import { ChatAttachmentsService } from "./chat-attachments.service"; +import { ChatGifsService } from "./chat-gifs.service"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; +import { ChatLobbyType } from "./enums/ChatLobbyTypes"; + +const ID = "0b7d6c1e-1111-4a2b-9c3d-000000000001"; +const ME = "76561198000000001"; +const OTHER = "76561198000000002"; + +describe("ChatMediaController", () => { + const chat = { + attachmentRefusal: jest.fn(), + canViewAttachment: jest.fn(), + }; + + const attachments = { + maxFileBytes: jest.fn(), + create: jest.fn(), + uploadPart: jest.fn(), + complete: jest.fn(), + setPoster: jest.fn(), + discard: jest.fn(), + find: jest.fn(), + stream: jest.fn(), + }; + + const gifs = { + enabled: jest.fn(), + search: jest.fn(), + }; + + let controller: ChatMediaController; + + const request = ( + steamId: string | undefined = ME, + overrides: Record = {}, + ) => + ({ + user: steamId + ? { steam_id: steamId, role: "user", name: "Someone" } + : undefined, + headers: {}, + is: jest.fn(() => "application/octet-stream"), + ...overrides, + }) as any; + + const body = (bytes: Buffer, contentType = "application/octet-stream") => + Object.assign(Readable.from([bytes]), { + user: { steam_id: ME, role: "user", name: "Someone" }, + headers: { "content-length": String(bytes.length) }, + is: (type: string) => (type === contentType ? type : false), + }) as any; + + const failure = async (promise: Promise) => { + try { + await promise; + } catch (error) { + expect(error).toBeInstanceOf(HttpException); + return { + status: (error as HttpException).getStatus(), + body: (error as HttpException).getResponse(), + }; + } + throw new Error("expected a refusal"); + }; + + beforeEach(() => { + jest.clearAllMocks(); + attachments.maxFileBytes.mockResolvedValue(100 * 1024 * 1024); + gifs.enabled.mockResolvedValue(true); + chat.attachmentRefusal.mockResolvedValue(null); + controller = new ChatMediaController( + chat as any, + attachments as any, + gifs as any, + ); + }); + + describe("config", () => { + it("tells the composer the operator's limits and whether GIFs are on", async () => { + await expect(controller.config()).resolves.toEqual({ + max_files: 4, + max_file_bytes: 100 * 1024 * 1024, + part_size: ChatAttachmentsService.PART_SIZE, + mime_types: Object.keys(ChatAttachmentsService.TYPES), + gifs: true, + }); + }); + + it("never hands out the GIPHY key", async () => { + const KEY = "giphy-secret-key"; + const realGifs = new ChatGifsService( + { log: jest.fn(), warn: jest.fn(), error: jest.fn() } as any, + { query: jest.fn(async () => [{ value: KEY }]) } as any, + { getConnection: () => ({}) } as any, + ); + + const config = await new ChatMediaController( + chat as any, + attachments as any, + realGifs, + ).config(); + + expect(config.gifs).toBe(true); + expect(JSON.stringify(config)).not.toContain(KEY); + }); + }); + + describe("starting an upload", () => { + const upload = { + type: ChatLobbyType.MatchMaking, + id: "lobby-1", + name: "smoke.png", + size: 1024, + mime_type: "image/png", + }; + + it("refuses a room the player may not post in, before storing anything", async () => { + chat.attachmentRefusal.mockResolvedValue(ChatErrorCode.NotAllowed); + + await expect( + failure(controller.create(request(), upload)), + ).resolves.toEqual({ + status: 403, + body: { code: ChatErrorCode.NotAllowed }, + }); + expect(attachments.create).not.toHaveBeenCalled(); + }); + + it("refuses a gagged player", async () => { + chat.attachmentRefusal.mockResolvedValue(ChatErrorCode.Gagged); + + await expect( + failure(controller.create(request(), upload)), + ).resolves.toMatchObject({ status: 403, body: { code: "gagged" } }); + }); + + it("refuses a room type it does not know", async () => { + await expect( + failure(controller.create(request(), { ...upload, type: "global" })), + ).resolves.toMatchObject({ status: 400 }); + expect(chat.attachmentRefusal).not.toHaveBeenCalled(); + }); + + it("says why a file was refused", async () => { + attachments.create.mockResolvedValue({ code: ChatErrorCode.TooLarge }); + + await expect( + failure(controller.create(request(), upload)), + ).resolves.toEqual({ status: 413, body: { code: "too_large" } }); + }); + + it("starts the upload as the signed in player", async () => { + attachments.create.mockResolvedValue({ id: ID, part_size: 1, parts: 1 }); + + await expect(controller.create(request(), upload)).resolves.toEqual({ + id: ID, + part_size: 1, + parts: 1, + }); + expect(attachments.create).toHaveBeenCalledWith( + ME, + ChatLobbyType.MatchMaking, + "lobby-1", + upload, + ); + }); + }); + + describe("parts", () => { + it("hands the request's stream to storage, with the length it declared", async () => { + attachments.uploadPart.mockResolvedValue(null); + const req = body(Buffer.from("part one")); + + await expect(controller.part(req, ID, "1")).resolves.toEqual({ + success: true, + }); + expect(attachments.uploadPart).toHaveBeenCalledWith(ME, ID, 1, req, 8); + }); + + it("refuses a part that does not say how long it is", async () => { + const req = body(Buffer.from("x")); + delete req.headers["content-length"]; + + await expect( + failure(controller.part(req, ID, "1")), + ).resolves.toMatchObject({ status: 411 }); + expect(attachments.uploadPart).not.toHaveBeenCalled(); + }); + + it("answers 429 when too many parts are already on their way", async () => { + attachments.uploadPart.mockResolvedValue(ChatErrorCode.RateLimited); + + await expect( + failure(controller.part(body(Buffer.from("x")), ID, "1")), + ).resolves.toEqual({ status: 429, body: { code: "rate_limited" } }); + }); + + // Anything else is read by a body parser first, and the stream this + // waits on has already ended. + it("only takes a raw body", async () => { + await expect( + failure( + controller.part(body(Buffer.from("{}"), "application/json"), ID, "1"), + ), + ).resolves.toMatchObject({ status: 415 }); + expect(attachments.uploadPart).not.toHaveBeenCalled(); + }); + + it("refuses a part bigger than any part can be, without reading it all", async () => { + const huge = body(Buffer.alloc(16)); + huge.headers["content-length"] = String( + ChatAttachmentsService.PART_SIZE + 1, + ); + + await expect( + failure(controller.part(huge, ID, "1")), + ).resolves.toMatchObject({ status: 413 }); + expect(attachments.uploadPart).not.toHaveBeenCalled(); + }); + + it("refuses an attachment id that is not one", async () => { + await expect( + failure(controller.part(body(Buffer.from("x")), "../x", "1")), + ).resolves.toMatchObject({ status: 404 }); + }); + + it("passes on why storage refused the part", async () => { + attachments.uploadPart.mockResolvedValue(ChatErrorCode.UnsupportedType); + + await expect( + failure(controller.part(body(Buffer.from("x")), ID, "1")), + ).resolves.toEqual({ status: 415, body: { code: "unsupported_type" } }); + }); + }); + + describe("posters", () => { + it("hands the poster's stream to storage, with the length it declared", async () => { + attachments.setPoster.mockResolvedValue(null); + const req = body(Buffer.from("RIFF0000WEBP")); + + await expect(controller.setPoster(req, ID)).resolves.toEqual({ + success: true, + }); + expect(attachments.setPoster).toHaveBeenCalledWith(ME, ID, req, 12); + }); + + it("refuses a poster bigger than a poster can be, without reading it", async () => { + const req = body(Buffer.alloc(16)); + req.headers["content-length"] = String( + ChatAttachmentsService.POSTER_MAX_BYTES + 1, + ); + + await expect( + failure(controller.setPoster(req, ID)), + ).resolves.toMatchObject({ status: 413 }); + expect(attachments.setPoster).not.toHaveBeenCalled(); + }); + }); + + describe("serving", () => { + const row = (overrides: Record = {}) => ({ + id: ID, + uploader_steam_id: ME, + room_type: ChatLobbyType.MatchMaking, + room_id: "lobby-1", + message_id: "m-1", + storage_prefix: `chat-attachments/rooms/2026-10-02/${ID}/`, + mime_type: "image/png", + poster_mime_type: null as string | null, + file_name: "smoke.png", + ...overrides, + }); + + const response = () => ({}) as any; + + it("answers 404 to anyone chat would not show the file to", async () => { + const req = request(OTHER); + attachments.find.mockResolvedValue(row()); + chat.canViewAttachment.mockResolvedValue(false); + + await expect( + failure(controller.file(req, response(), ID)), + ).resolves.toMatchObject({ status: 404 }); + expect(chat.canViewAttachment).toHaveBeenCalledWith(row(), req.user); + expect(attachments.stream).not.toHaveBeenCalled(); + }); + + it("answers 404 for a file there is no row for", async () => { + attachments.find.mockResolvedValue(undefined); + + await expect( + failure(controller.file(request(ME), response(), ID)), + ).resolves.toMatchObject({ status: 404 }); + expect(chat.canViewAttachment).not.toHaveBeenCalled(); + }); + + it("streams the file under a name that matches what it is", async () => { + const req = request(OTHER); + const res = response(); + attachments.find.mockResolvedValue(row({ mime_type: "image/jpeg" })); + chat.canViewAttachment.mockResolvedValue(true); + + await controller.file(req, res, ID); + + expect(attachments.stream).toHaveBeenCalledWith( + `chat-attachments/rooms/2026-10-02/${ID}/file`, + "image/jpeg", + "smoke.jpg", + req, + res, + ); + }); + + it("answers 404 for a video without a poster", async () => { + attachments.find.mockResolvedValue(row({ mime_type: "video/mp4" })); + chat.canViewAttachment.mockResolvedValue(true); + + await expect( + failure(controller.poster(request(ME), response(), ID)), + ).resolves.toMatchObject({ status: 404 }); + }); + + it("serves a poster as the image it is", async () => { + attachments.find.mockResolvedValue( + row({ mime_type: "video/mp4", poster_mime_type: "image/webp" }), + ); + chat.canViewAttachment.mockResolvedValue(true); + + await controller.poster(request(ME), response(), ID); + + expect(attachments.stream).toHaveBeenCalledWith( + `chat-attachments/rooms/2026-10-02/${ID}/poster`, + "image/webp", + "poster.webp", + expect.anything(), + expect.anything(), + ); + }); + }); + + describe("GIF search", () => { + it("answers 404 when GIFs are off", async () => { + gifs.search.mockResolvedValue("disabled"); + + await expect( + failure(controller.searchGifs(request(), "gg", "0")), + ).resolves.toMatchObject({ status: 404, body: { code: "disabled" } }); + }); + + it("answers 429 to a player searching too fast", async () => { + gifs.search.mockResolvedValue("rate_limited"); + + await expect( + failure(controller.searchGifs(request(), "gg", "0")), + ).resolves.toMatchObject({ status: 429 }); + }); + + it("says GIFs are busy when the panel's GIPHY allowance is spent", async () => { + gifs.search.mockResolvedValue("busy"); + + await expect( + failure(controller.searchGifs(request(), "gg", "0")), + ).resolves.toEqual({ status: 503, body: { code: "busy" } }); + }); + + it("searches as the signed in player", async () => { + gifs.search.mockResolvedValue({ results: [], next: null }); + + await expect( + controller.searchGifs(request(), "gg", "24"), + ).resolves.toEqual({ results: [], next: null }); + expect(gifs.search).toHaveBeenCalledWith(ME, "gg", 24); + }); + }); +}); diff --git a/src/chat/chat-media.controller.ts b/src/chat/chat-media.controller.ts new file mode 100644 index 00000000..f569daff --- /dev/null +++ b/src/chat/chat-media.controller.ts @@ -0,0 +1,315 @@ +import { + Body, + Controller, + Delete, + Get, + HttpException, + HttpStatus, + Param, + Post, + Put, + Query, + Req, + Res, + UseGuards, +} from "@nestjs/common"; +import { Request, Response } from "express"; +import { SteamGuard } from "src/auth/strategies/SteamGuard"; +import { User } from "src/auth/types/User"; +import { ChatService } from "./chat.service"; +import { ChatAttachmentsService } from "./chat-attachments.service"; +import { ChatGifsService } from "./chat-gifs.service"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; +import { ChatLobbyType } from "./enums/ChatLobbyTypes"; + +// Uploads go through the api in parts rather than to storage directly: each +// part is a request well under Cloudflare's 100 MB cap, and nothing needs the +// bucket to answer a browser's CORS preflight. +@Controller("chat") +export class ChatMediaController { + private static readonly UUID = + /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + + private static readonly STATUS: Partial> = { + [ChatErrorCode.NotAllowed]: HttpStatus.FORBIDDEN, + [ChatErrorCode.Gagged]: HttpStatus.FORBIDDEN, + [ChatErrorCode.NotFound]: HttpStatus.NOT_FOUND, + [ChatErrorCode.Disabled]: HttpStatus.NOT_FOUND, + [ChatErrorCode.TooLarge]: HttpStatus.PAYLOAD_TOO_LARGE, + [ChatErrorCode.UnsupportedType]: HttpStatus.UNSUPPORTED_MEDIA_TYPE, + [ChatErrorCode.TooManyPending]: HttpStatus.TOO_MANY_REQUESTS, + [ChatErrorCode.QuotaExceeded]: HttpStatus.TOO_MANY_REQUESTS, + [ChatErrorCode.RateLimited]: HttpStatus.TOO_MANY_REQUESTS, + [ChatErrorCode.Unavailable]: HttpStatus.BAD_GATEWAY, + [ChatErrorCode.Busy]: HttpStatus.SERVICE_UNAVAILABLE, + }; + + constructor( + private readonly chat: ChatService, + private readonly attachments: ChatAttachmentsService, + private readonly gifs: ChatGifsService, + ) {} + + private static fail(code: ChatErrorCode): never { + throw new HttpException( + { code }, + ChatMediaController.STATUS[code] ?? HttpStatus.BAD_REQUEST, + ); + } + + private static attachmentId(id: string): string { + if (!ChatMediaController.UUID.test(id)) { + ChatMediaController.fail(ChatErrorCode.NotFound); + } + + return id; + } + + // Anything but a raw body is read by one of the app's body parsers first, + // and the stream handed on would already have ended. The length is what the + // body is held to, so a request that does not give one is refused. + private static rawLength(request: Request, max: number): number { + if (!request.is("application/octet-stream")) { + throw new HttpException( + { code: ChatErrorCode.UnsupportedType }, + HttpStatus.UNSUPPORTED_MEDIA_TYPE, + ); + } + + const declared = Number(request.headers["content-length"]); + + if (!Number.isSafeInteger(declared) || declared <= 0) { + throw new HttpException( + { code: ChatErrorCode.Invalid }, + HttpStatus.LENGTH_REQUIRED, + ); + } + + if (declared > max) { + ChatMediaController.fail(ChatErrorCode.TooLarge); + } + + return declared; + } + + @Get("attachments/config") + @UseGuards(SteamGuard) + public async config() { + return { + max_files: ChatAttachmentsService.MAX_PER_MESSAGE, + max_file_bytes: await this.attachments.maxFileBytes(), + part_size: ChatAttachmentsService.PART_SIZE, + mime_types: Object.keys(ChatAttachmentsService.TYPES), + gifs: await this.gifs.enabled(), + }; + } + + @Post("attachments") + @UseGuards(SteamGuard) + public async create( + @Req() request: Request, + @Body() + body: { + type?: unknown; + id?: unknown; + name?: unknown; + size?: unknown; + mime_type?: unknown; + width?: unknown; + height?: unknown; + duration_ms?: unknown; + }, + ) { + const user = request.user as User; + + if ( + !Object.values(ChatLobbyType).includes(body?.type as ChatLobbyType) || + typeof body.id !== "string" + ) { + ChatMediaController.fail(ChatErrorCode.Invalid); + } + + const type = body.type as ChatLobbyType; + + const refusal = await this.chat.attachmentRefusal(type, body.id, user); + + if (refusal) { + ChatMediaController.fail(refusal); + } + + const created = await this.attachments.create( + String(user.steam_id), + type, + body.id, + body, + ); + + if ("code" in created) { + ChatMediaController.fail(created.code); + } + + return created; + } + + @Put("attachments/:id/parts/:part") + @UseGuards(SteamGuard) + public async part( + @Req() request: Request, + @Param("id") id: string, + @Param("part") part: string, + ) { + const attachmentId = ChatMediaController.attachmentId(id); + + const length = ChatMediaController.rawLength( + request, + ChatAttachmentsService.PART_SIZE, + ); + + const refusal = await this.attachments.uploadPart( + String((request.user as User).steam_id), + attachmentId, + Number(part), + request, + length, + ); + + if (refusal) { + ChatMediaController.fail(refusal); + } + + return { success: true }; + } + + @Post("attachments/:id/complete") + @UseGuards(SteamGuard) + public async complete(@Req() request: Request, @Param("id") id: string) { + const completed = await this.attachments.complete( + String((request.user as User).steam_id), + ChatMediaController.attachmentId(id), + ); + + if ("code" in completed) { + ChatMediaController.fail(completed.code); + } + + return completed; + } + + @Put("attachments/:id/poster") + @UseGuards(SteamGuard) + public async setPoster(@Req() request: Request, @Param("id") id: string) { + const attachmentId = ChatMediaController.attachmentId(id); + + const length = ChatMediaController.rawLength( + request, + ChatAttachmentsService.POSTER_MAX_BYTES, + ); + + const refusal = await this.attachments.setPoster( + String((request.user as User).steam_id), + attachmentId, + request, + length, + ); + + if (refusal) { + ChatMediaController.fail(refusal); + } + + return { success: true }; + } + + @Delete("attachments/:id") + @UseGuards(SteamGuard) + public async discard(@Req() request: Request, @Param("id") id: string) { + await this.attachments.discard( + String((request.user as User).steam_id), + ChatMediaController.attachmentId(id), + ); + + return { success: true }; + } + + @Get("attachments/:id") + @UseGuards(SteamGuard) + public async file( + @Req() request: Request, + @Res() response: Response, + @Param("id") id: string, + ) { + const row = await this.viewable(request, id); + + await this.attachments.stream( + `${row.storage_prefix}file`, + row.mime_type, + ChatAttachmentsService.downloadName(row.file_name, row.mime_type), + request, + response, + ); + } + + @Get("attachments/:id/poster") + @UseGuards(SteamGuard) + public async poster( + @Req() request: Request, + @Res() response: Response, + @Param("id") id: string, + ) { + const row = await this.viewable(request, id); + + if (!row.poster_mime_type) { + ChatMediaController.fail(ChatErrorCode.NotFound); + } + + await this.attachments.stream( + `${row.storage_prefix}poster`, + row.poster_mime_type, + `poster.${row.poster_mime_type.split("/")[1]}`, + request, + response, + ); + } + + // 404 rather than 403, so asking never confirms a file exists. + private async viewable(request: Request, id: string) { + const row = await this.attachments.find( + ChatMediaController.attachmentId(id), + ); + + if ( + !row || + !(await this.chat.canViewAttachment(row, request.user as User)) + ) { + ChatMediaController.fail(ChatErrorCode.NotFound); + } + + return row; + } + + @Get("gifs") + @UseGuards(SteamGuard) + public async searchGifs( + @Req() request: Request, + @Query("q") query?: string, + @Query("offset") offset?: string, + ) { + const page = await this.gifs.search( + String((request.user as User).steam_id), + typeof query === "string" ? query : "", + Number(offset ?? 0), + ); + + switch (page) { + case "disabled": + return ChatMediaController.fail(ChatErrorCode.Disabled); + case "rate_limited": + return ChatMediaController.fail(ChatErrorCode.RateLimited); + case "unavailable": + return ChatMediaController.fail(ChatErrorCode.Unavailable); + case "busy": + return ChatMediaController.fail(ChatErrorCode.Busy); + default: + return page; + } + } +} diff --git a/src/chat/chat-media.spec.ts b/src/chat/chat-media.spec.ts new file mode 100644 index 00000000..00d4d33b --- /dev/null +++ b/src/chat/chat-media.spec.ts @@ -0,0 +1,817 @@ +import { ChatService } from "./chat.service"; +import { ChatErrorCode } from "./enums/ChatErrorCode"; +import { ChatLobbyType } from "./enums/ChatLobbyTypes"; +import { directRoomId } from "./utilities/directRoomId"; + +const ME = "76561198000000001"; +const FRIEND = "76561198000000002"; + +const ID_1 = "0b7d6c1e-1111-4a2b-9c3d-000000000001"; +const ID_2 = "0b7d6c1e-1111-4a2b-9c3d-000000000002"; +const ID_3 = "0b7d6c1e-1111-4a2b-9c3d-000000000003"; +const ID_4 = "0b7d6c1e-1111-4a2b-9c3d-000000000004"; +const ID_5 = "0b7d6c1e-1111-4a2b-9c3d-000000000005"; + +const descriptor = (id: string) => ({ + id, + kind: "image", + name: `${id}.png`, + mime_type: "image/png", + size: 1024, + width: 640, + height: 360, +}); + +describe("ChatService attachments and GIFs", () => { + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() }; + + const redis = { + hset: jest.fn(), + hget: jest.fn(), + hgetall: jest.fn().mockResolvedValue({}), + hdel: jest.fn().mockResolvedValue(1), + get: jest.fn().mockResolvedValue(null), + set: jest.fn(), + del: jest.fn(), + keys: jest.fn().mockResolvedValue([]), + expire: jest.fn(), + publish: jest.fn(), + sendCommand: jest.fn(), + eval: jest.fn(), + }; + + let gagged: boolean; + let queries: Array<{ sql: string; bindings: any[] }>; + + const answer = async (sql: string, bindings: any[] = []): Promise => { + queries.push({ sql, bindings }); + + if (sql.includes("public.is_gagged")) { + return [{ gagged }]; + } + + if (sql.includes("INSERT INTO public.direct_messages")) { + return [{ id: bindings[0] }]; + } + + return []; + }; + + const postgres = { + query: jest.fn(answer), + transaction: jest.fn(async (work: (client: any) => Promise) => + work({ + query: async (sql: string, bindings: any[]) => ({ + rows: await postgres.query(sql, bindings), + }), + }), + ), + }; + + const hasura = { + query: jest.fn(async (query: any) => { + if (query.lobby_players_by_pk) { + return { lobby_players_by_pk: { status: "Accepted" } }; + } + + if (query.lobby_players) { + return { + lobby_players: [{ steam_id: ME }, { steam_id: FRIEND }], + }; + } + + if (query.matches_by_pk) { + return { + matches_by_pk: { + is_coach: false, + is_organizer: false, + is_in_lineup: true, + }, + }; + } + + if (query.match_lineups_by_pk) { + return { + match_lineups_by_pk: { + match_id: "m-1", + coach_steam_id: null, + is_on_lineup: true, + }, + }; + } + + if (query.friends) { + return { friends: [{ status: "Accepted" }] }; + } + + if (query.players_by_pk) { + return { + players_by_pk: { steam_id: ME, name: "Someone", role: "user" }, + }; + } + + return {}; + }), + }; + + const playerBlocks = { + hasBlocked: jest.fn().mockResolvedValue(false), + isBlockedEitherWay: jest.fn().mockResolvedValue(false), + blockedBy: jest.fn().mockResolvedValue(new Set()), + blockedAmong: jest.fn().mockResolvedValue(new Map()), + }; + + const push = { + sendChatMessage: jest.fn().mockResolvedValue(undefined), + retractChatMessage: jest.fn().mockResolvedValue(undefined), + editChatMessage: jest.fn().mockResolvedValue(undefined), + }; + + const attachments = { + claim: jest.fn(), + sentBy: jest.fn().mockResolvedValue(false), + expireMessage: jest.fn().mockResolvedValue(undefined), + markDeleted: jest.fn().mockResolvedValue(undefined), + moveRoom: jest.fn().mockResolvedValue(undefined), + }; + + const gifs = { + enabled: jest.fn(), + }; + + let service: ChatService; + + const player = () => + ({ + steam_id: ME, + name: "Someone", + role: "user", + avatar_url: "avatar", + profile_url: "profile", + }) as any; + + const stored = (key: string) => + redis.hset.mock.calls + .filter(([hash]) => hash === key) + .map(([, , value]) => JSON.parse(value)); + + const flush = () => new Promise((resolve) => setImmediate(resolve)); + + const send = ( + type: ChatLobbyType, + id: string, + message: string, + media: { attachments?: unknown; gif?: unknown }, + ) => + service.sendMessageToChat(type, id, player(), message, false, "web", media); + + beforeEach(() => { + jest.clearAllMocks(); + gagged = false; + queries = []; + redis.get.mockResolvedValue(null); + redis.set.mockResolvedValue("OK"); + postgres.query.mockImplementation(answer); + redis.hget.mockResolvedValue(JSON.stringify({ user: { steam_id: ME } })); + redis.eval.mockImplementation(async (script: string) => + script.includes("INCR") ? 1 : [1, 1], + ); + attachments.claim.mockImplementation(async (ids: string[]) => + ids.map(descriptor), + ); + gifs.enabled.mockResolvedValue(true); + + service = new ChatService( + logger as any, + { connect: jest.fn() } as any, + hasura as any, + postgres as any, + { getConnection: () => redis } as any, + push as any, + playerBlocks as any, + attachments as any, + gifs as any, + ); + }); + + describe("attachments", () => { + it("sends files to a lobby with no text, carried on the message", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1, ID_2], + }), + ).resolves.toEqual({ accepted: true, messageId: expect.any(String) }); + + const [message] = stored("chat_matchmaking_lobby-1"); + + expect(message.message).toBe(""); + expect(message.attachments).toEqual([descriptor(ID_1), descriptor(ID_2)]); + }); + + it("claims the files for this sender, this room and this message", async () => { + const before = Date.now(); + + const result = await send( + ChatLobbyType.MatchMaking, + "lobby-1", + "smokes", + { + attachments: [ID_1], + }, + ); + + const [ids, claim] = attachments.claim.mock.calls[0]; + + expect(ids).toEqual([ID_1]); + expect(claim).toMatchObject({ + type: ChatLobbyType.MatchMaking, + roomId: "lobby-1", + steamId: ME, + messageId: result.accepted ? result.messageId : "", + }); + // Lives as long as a lobby keeps its messages, and a little longer so + // a file is never gone while its message still shows. + expect(claim.expiresAt.getTime()).toBeGreaterThanOrEqual( + before + 3600 * 1000, + ); + expect(claim.expiresAt.getTime()).toBeLessThanOrEqual( + Date.now() + 3600 * 1000 + 15 * 60 * 1000, + ); + }); + + it("refuses files the sender could not claim, and stores nothing", async () => { + attachments.claim.mockResolvedValue(null); + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + + expect(redis.hset).not.toHaveBeenCalled(); + expect(push.sendChatMessage).not.toHaveBeenCalled(); + }); + + // A send whose answer was lost is retried with the same files; the first + // one landed, so the retry says so instead of failing. + it("says the files were already sent when the same sender sent them here", async () => { + attachments.claim.mockResolvedValue(null); + attachments.sentBy.mockResolvedValue(true); + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.AlreadySent }); + + expect(attachments.sentBy).toHaveBeenCalledWith( + [ID_1], + expect.objectContaining({ + steamId: ME, + type: ChatLobbyType.MatchMaking, + roomId: "lobby-1", + }), + ); + }); + + it("says the same of a direct message's files", async () => { + attachments.claim.mockResolvedValue(null); + attachments.sentBy.mockResolvedValue(true); + + await expect( + send(ChatLobbyType.Direct, directRoomId(ME, FRIEND), "", { + attachments: [ID_1], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.AlreadySent }); + }); + + // Both rooms are relayed into the game server, which shows only text. + it.each([ + [ChatLobbyType.Match, "m-1"], + [ChatLobbyType.MatchTeam, "m-1:l-1"], + ])("keeps %s text-only", async (type, id) => { + await expect( + send(type, id, "look", { attachments: [ID_1] }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.NotAllowed }); + + expect(attachments.claim).not.toHaveBeenCalled(); + expect(redis.hset).not.toHaveBeenCalled(); + }); + + it("claims ids in the case postgres hands them back in", async () => { + await send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1.toUpperCase()], + }); + + expect(attachments.claim.mock.calls[0][0]).toEqual([ID_1]); + expect(stored("chat_matchmaking_lobby-1")[0].attachments).toEqual([ + descriptor(ID_1), + ]); + }); + + it("refuses the same file twice, whatever its case", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1, ID_1.toUpperCase()], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + }); + + it("never leaves files claimed by a room message that was not stored", async () => { + const committed = jest.fn(); + postgres.transaction.mockImplementationOnce(async (work: any) => { + const result = await work({ + query: async (sql: string, bindings: any[]) => ({ + rows: await postgres.query(sql, bindings), + }), + }); + committed(); + return result; + }); + redis.hset.mockRejectedValueOnce(new Error("redis went away")); + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }), + ).rejects.toThrow("redis went away"); + + expect(attachments.claim.mock.calls[0][2]).toBeDefined(); + expect(committed).not.toHaveBeenCalled(); + }); + + it("takes the message back out of the room when its expiry cannot be set", async () => { + redis.sendCommand.mockRejectedValueOnce(new Error("HEXPIRE refused")); + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }), + ).rejects.toThrow("HEXPIRE refused"); + + const [key, field] = redis.hset.mock.calls[0]; + expect(redis.hdel).toHaveBeenCalledWith(key, field); + }); + + // The message is already in the room by the time the claim commits, so a + // commit that fails would leave it pointing at files it never claimed. + it("takes the message back out of the room when the claim fails to commit", async () => { + postgres.transaction.mockImplementationOnce(async (work: any) => { + await work({ + query: async (sql: string, bindings: any[]) => ({ + rows: await postgres.query(sql, bindings), + }), + }); + throw new Error("commit failed"); + }); + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }), + ).rejects.toThrow("commit failed"); + + const [key, field] = redis.hset.mock.calls[0]; + expect(redis.hdel).toHaveBeenCalledWith(key, field); + }); + + it("refuses more than four", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1, ID_2, ID_3, ID_4, ID_5], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + + expect(attachments.claim).not.toHaveBeenCalled(); + }); + + it.each([ + ["the same file twice", [ID_1, ID_1]], + ["something that is not an id", ["../../etc"]], + ["a number", [5]], + ["not a list", ID_1], + ])("refuses %s", async (_, list) => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { attachments: list }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + + expect(attachments.claim).not.toHaveBeenCalled(); + }); + + it("still refuses an empty message with nothing attached", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", " ", { attachments: [] }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + }); + + it("holds a gagged player to the same rule as their text", async () => { + gagged = true; + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Gagged }); + + expect(attachments.claim).not.toHaveBeenCalled(); + }); + + it("lets a gagged player attach to a friend, as they may write to one", async () => { + gagged = true; + const room = directRoomId(ME, FRIEND); + + await expect( + send(ChatLobbyType.Direct, room, "", { attachments: [ID_1] }), + ).resolves.toEqual({ accepted: true, messageId: expect.any(String) }); + }); + + it("claims a direct message's files with the message, for as long as it lasts", async () => { + const room = directRoomId(ME, FRIEND); + + const result = await send(ChatLobbyType.Direct, room, "", { + attachments: [ID_1], + }); + + const [ids, claim, client] = attachments.claim.mock.calls[0]; + + expect(ids).toEqual([ID_1]); + expect(claim).toMatchObject({ + type: ChatLobbyType.Direct, + roomId: room, + steamId: ME, + messageId: result.accepted ? result.messageId : "", + expiresAt: null, + }); + // Inside the insert's transaction, so a refused insert takes the claim + // back with it. + expect(client).toBeDefined(); + + const insert = queries.find(({ sql }) => + sql.includes("INSERT INTO public.direct_messages"), + ); + expect(insert.bindings).toContain(JSON.stringify([descriptor(ID_1)])); + }); + + it("pushes 'Attachment' when there is nothing typed", async () => { + await send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1], + }); + await flush(); + + expect(push.sendChatMessage).toHaveBeenCalledWith( + [FRIEND], + expect.objectContaining({ message: "Attachment" }), + ); + }); + + it("counts several attachments in the push", async () => { + await send(ChatLobbyType.MatchMaking, "lobby-1", "", { + attachments: [ID_1, ID_2, ID_3], + }); + await flush(); + + expect(push.sendChatMessage).toHaveBeenCalledWith( + [FRIEND], + expect.objectContaining({ message: "3 attachments" }), + ); + }); + + it("pushes the text when there is some", async () => { + await send(ChatLobbyType.MatchMaking, "lobby-1", "smokes", { + attachments: [ID_1], + }); + await flush(); + + expect(push.sendChatMessage).toHaveBeenCalledWith( + [FRIEND], + expect.objectContaining({ message: "smokes" }), + ); + }); + }); + + describe("GIFs", () => { + const GIF = { id: "abc123", width: 480, height: 270 }; + + it("sends a GIF as a reference to GIPHY, not an upload", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { gif: GIF }), + ).resolves.toEqual({ accepted: true, messageId: expect.any(String) }); + + const [message] = stored("chat_matchmaking_lobby-1"); + + expect(message.gif).toEqual(GIF); + expect(message.attachments).toBeUndefined(); + expect(attachments.claim).not.toHaveBeenCalled(); + }); + + it("stores only the id and size, whatever else came with it", async () => { + await send(ChatLobbyType.MatchMaking, "lobby-1", "", { + gif: { ...GIF, url: "https://evil.example/pixel.gif" }, + }); + + expect(stored("chat_matchmaking_lobby-1")[0].gif).toEqual(GIF); + }); + + it("refuses a GIF when the operator has not set a GIPHY key", async () => { + gifs.enabled.mockResolvedValue(false); + + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { gif: GIF }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.NotAllowed }); + + expect(redis.hset).not.toHaveBeenCalled(); + }); + + it("keeps match chat text-only", async () => { + await expect( + send(ChatLobbyType.Match, "m-1", "", { gif: GIF }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.NotAllowed }); + }); + + it("refuses something that is not a GIPHY id", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + gif: { id: "https://evil.example/x.gif", width: 1, height: 1 }, + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + }); + + it("refuses a GIF and files in one message", async () => { + await expect( + send(ChatLobbyType.MatchMaking, "lobby-1", "", { + gif: GIF, + attachments: [ID_1], + }), + ).resolves.toEqual({ accepted: false, code: ChatErrorCode.Invalid }); + }); + + it("stores a direct message's GIF with it", async () => { + await send(ChatLobbyType.Direct, directRoomId(ME, FRIEND), "", { + gif: GIF, + }); + + const insert = queries.find(({ sql }) => + sql.includes("INSERT INTO public.direct_messages"), + ); + expect(insert.bindings).toContain(JSON.stringify(GIF)); + }); + + it("pushes 'GIF' when there is nothing typed", async () => { + await send(ChatLobbyType.MatchMaking, "lobby-1", "", { gif: GIF }); + await flush(); + + expect(push.sendChatMessage).toHaveBeenCalledWith( + [FRIEND], + expect.objectContaining({ message: "GIF" }), + ); + }); + }); + + describe("who may open a file", () => { + const row = (overrides: Record = {}) => + ({ + id: ID_1, + uploader_steam_id: FRIEND, + room_type: ChatLobbyType.MatchMaking, + room_id: "lobby-1", + message_id: "3f0c1d2e-4b5a-4c6d-8e7f-9a0b1c2d3e4f", + deleted_at: "2026-10-02T12:00:00.000Z", + ...overrides, + }) as any; + + // The session still says moderator; the players table says otherwise. + it("judges by the player's role now, not the one their session holds", async () => { + await expect( + service.canViewAttachment(row(), { + steam_id: ME, + role: "moderator", + } as any), + ).resolves.toBe(false); + + expect(hasura.query).toHaveBeenCalledWith( + expect.objectContaining({ players_by_pk: expect.anything() }), + ); + }); + + // Every image, every revalidation and every range of a video asks. + it("looks the player up once a minute, not on every request", async () => { + const cache = new Map(); + redis.get.mockImplementation( + async (key: string) => cache.get(key) ?? null, + ); + redis.set.mockImplementation(async (key: string, value: string) => { + cache.set(key, value); + return "OK"; + }); + + for (let i = 0; i < 3; i++) { + await service.canViewAttachment(row({ deleted_at: null }), { + steam_id: ME, + role: "user", + } as any); + } + + expect( + hasura.query.mock.calls.filter(([query]) => query.players_by_pk), + ).toHaveLength(1); + expect( + redis.set.mock.calls.find(([key]) => String(key).includes(ME)), + ).toEqual(expect.arrayContaining(["EX", 60])); + }); + + it("lets someone in the room open a sent file", async () => { + await expect( + service.canViewAttachment(row({ deleted_at: null }), { + steam_id: ME, + role: "user", + } as any), + ).resolves.toBe(true); + }); + }); + + describe("removing", () => { + const MESSAGE_ID = "3f0c1d2e-4b5a-4c6d-8e7f-9a0b1c2d3e4f"; + + const storeInLobby = (media: Record) => + redis.hget.mockImplementation(async (key: string) => + key === "chat_matchmaking_lobby-1" + ? JSON.stringify({ + id: MESSAGE_ID, + message: "", + timestamp: new Date().toISOString(), + source: "web", + from: { role: "user", name: "Someone", steam_id: ME }, + ...media, + }) + : JSON.stringify({ user: { steam_id: ME } }), + ); + + const audit = () => + queries.find(({ sql }) => + sql.includes("INSERT INTO public.chat_message_deletions"), + ); + + // Posting abuse and deleting it must still leave staff something to see. + it("keeps a deleted room message's files as evidence, out of the room's sight", async () => { + storeInLobby({ attachments: [descriptor(ID_1)] }); + + await expect( + service.deleteMessage( + ChatLobbyType.MatchMaking, + "lobby-1", + MESSAGE_ID, + player(), + ), + ).resolves.toEqual({ deleted: true }); + + expect(attachments.markDeleted).toHaveBeenCalledWith( + ChatLobbyType.MatchMaking, + "lobby-1", + MESSAGE_ID, + ); + expect(attachments.expireMessage).not.toHaveBeenCalled(); + }); + + it("records a deleted message's files and GIF in the audit", async () => { + storeInLobby({ attachments: [descriptor(ID_1)] }); + + await service.deleteMessage( + ChatLobbyType.MatchMaking, + "lobby-1", + MESSAGE_ID, + player(), + ); + + expect(audit().sql).toMatch(/attachments, gif/); + expect(audit().bindings).toContain(JSON.stringify([descriptor(ID_1)])); + + queries = []; + storeInLobby({ gif: { id: "abc123", width: 480, height: 270 } }); + + await service.deleteMessage( + ChatLobbyType.MatchMaking, + "lobby-1", + MESSAGE_ID, + player(), + ); + + expect(audit().bindings).toContain( + JSON.stringify({ id: "abc123", width: 480, height: 270 }), + ); + }); + + it("deletes a direct message's files as soon as the message goes", async () => { + const room = directRoomId(ME, FRIEND); + postgres.query.mockImplementation( + async (sql: string, bindings: any[] = []) => { + queries.push({ sql, bindings }); + + if (sql.includes("AS open")) { + return [{ author: ME, open: true }]; + } + + if (sql.includes("DELETE FROM public.direct_messages")) { + return [{ id: MESSAGE_ID }]; + } + + return []; + }, + ); + + await expect( + service.deleteMessage(ChatLobbyType.Direct, room, MESSAGE_ID, player()), + ).resolves.toEqual({ deleted: true }); + + expect(attachments.expireMessage).toHaveBeenCalledWith( + ChatLobbyType.Direct, + room, + MESSAGE_ID, + ); + }); + + // A conversation between friends is not moderated, so there is no + // evidence to keep. + it("still deletes a direct message's files straight away", async () => { + const room = directRoomId(ME, FRIEND); + postgres.query.mockImplementation( + async (sql: string, bindings: any[] = []) => { + queries.push({ sql, bindings }); + + if (sql.includes("AS open")) { + return [{ author: ME, open: true }]; + } + + if (sql.includes("DELETE FROM public.direct_messages")) { + return [{ id: MESSAGE_ID }]; + } + + return []; + }, + ); + + await service.deleteMessage( + ChatLobbyType.Direct, + room, + MESSAGE_ID, + player(), + ); + + expect(attachments.markDeleted).not.toHaveBeenCalled(); + }); + + // The match chat archive keeps every line the match room holds, moved + // lines included; their files move with them. + it("archives a draft's moved lines and moves their files", async () => { + const line = { + id: MESSAGE_ID, + message: "gl", + timestamp: new Date().toISOString(), + source: "web", + attachments: [descriptor(ID_1)], + from: { role: "user", name: "Someone", steam_id: ME }, + }; + redis.eval.mockImplementation(async (script: string) => + script.includes("HGETALL") ? [MESSAGE_ID, JSON.stringify(line)] : 1, + ); + + await service.migrateLobbyMessages( + ChatLobbyType.Draft, + "draft-1", + ChatLobbyType.Match, + "m-1", + ); + + expect(attachments.moveRoom).toHaveBeenCalledWith( + ChatLobbyType.Draft, + "draft-1", + ChatLobbyType.Match, + "m-1", + expect.any(Date), + ); + expect( + redis.eval.mock.calls.some( + ([, , key, id]) => String(key).includes("m-1") && id === MESSAGE_ID, + ), + ).toBe(true); + }); + + it("carries a draft's files into the match its chat moves to", async () => { + redis.eval.mockResolvedValue(0); + + await service.migrateLobbyMessages( + ChatLobbyType.Draft, + "draft-1", + ChatLobbyType.Match, + "m-1", + ); + + expect(attachments.moveRoom).toHaveBeenCalledWith( + ChatLobbyType.Draft, + "draft-1", + ChatLobbyType.Match, + "m-1", + expect.any(Date), + ); + }); + }); +}); diff --git a/src/chat/chat.gateway.spec.ts b/src/chat/chat.gateway.spec.ts index 219d69d5..2d4a0622 100644 --- a/src/chat/chat.gateway.spec.ts +++ b/src/chat/chat.gateway.spec.ts @@ -94,8 +94,80 @@ describe("ChatGateway lobby:chat", () => { "t-1", expect.objectContaining({ steam_id: "1" }), "hello", + false, + "web", + undefined, + ); + }); + + it("sends attachments that came without any text", async () => { + await gateway.lobby( + { + id: "lobby-1", + type: ChatLobbyType.MatchMaking, + message: "", + attachments: ["a-1", "a-2"], + }, + client(), + ); + + expect(chat.sendMessageToChat).toHaveBeenCalledWith( + ChatLobbyType.MatchMaking, + "lobby-1", + expect.objectContaining({ steam_id: "1" }), + "", + false, + "web", + { attachments: ["a-1", "a-2"], gif: undefined }, + ); + }); + + it("sends a GIF that came without any text", async () => { + const gif = { id: "abc123", width: 480, height: 270 }; + + await gateway.lobby( + { id: "lobby-1", type: ChatLobbyType.MatchMaking, gif } as any, + client(), + ); + + expect(chat.sendMessageToChat).toHaveBeenCalledWith( + ChatLobbyType.MatchMaking, + "lobby-1", + expect.objectContaining({ steam_id: "1" }), + "", + false, + "web", + { attachments: undefined, gif }, ); }); + + it("tells the sender a captioned upload is too long", async () => { + const socket = client(); + + await gateway.lobby( + { + id: "lobby-1", + type: ChatLobbyType.MatchMaking, + message: "a".repeat(ChatService.MAX_MESSAGE_LENGTH + 1), + attachments: ["a-1"], + requestId: "r-9", + }, + socket, + ); + + expect(chat.sendMessageToChat).not.toHaveBeenCalled(); + expect(sent(socket)).toEqual([ + { + event: "chat:error", + data: { + code: ChatErrorCode.TooLong, + action: "send", + max: ChatService.MAX_MESSAGE_LENGTH, + requestId: "r-9", + }, + }, + ]); + }); }); describe("length", () => { @@ -160,6 +232,9 @@ describe("ChatGateway lobby:chat", () => { "m-1", expect.anything(), message, + false, + "web", + undefined, ); }); }); diff --git a/src/chat/chat.gateway.ts b/src/chat/chat.gateway.ts index 21d6537b..592c630e 100644 --- a/src/chat/chat.gateway.ts +++ b/src/chat/chat.gateway.ts @@ -98,9 +98,11 @@ export class ChatGateway { @MessageBody() data: { id: string; - message: unknown; + message?: unknown; type: ChatLobbyType; requestId?: string; + attachments?: unknown; + gif?: unknown; }, @ConnectedSocket() client: FiveStackWebSocketClient, ) { @@ -117,7 +119,15 @@ export class ChatGateway { const requestId = typeof data.requestId === "string" ? data.requestId : undefined; - const parsed = ChatService.messageText(data.message); + const media = + data.attachments !== undefined || data.gif !== undefined + ? { attachments: data.attachments, gif: data.gif } + : undefined; + + const parsed = ChatService.messageText( + data.message, + ChatService.hasMedia(media), + ); if ("error" in parsed) { if (parsed.error === ChatErrorCode.TooLong) { @@ -131,6 +141,9 @@ export class ChatGateway { data.id, client.user, parsed.text, + false, + "web", + media, ); // Only a message the room accepted may reach the game server: the relay diff --git a/src/chat/chat.module.ts b/src/chat/chat.module.ts index 8b155c7a..d2f716fd 100644 --- a/src/chat/chat.module.ts +++ b/src/chat/chat.module.ts @@ -17,6 +17,12 @@ import { PlayerBlocksModule } from "src/player-blocks/player-blocks.module"; import { ChatQueues } from "./enums/ChatQueues"; import { PruneDirectMessages } from "./jobs/PruneDirectMessages"; import { BackfillDirectMessages } from "./jobs/BackfillDirectMessages"; +import { RemoveExpiredChatAttachments } from "./jobs/RemoveExpiredChatAttachments"; +import { SweepChatAttachments } from "./jobs/SweepChatAttachments"; +import { ChatAttachmentsService } from "./chat-attachments.service"; +import { ChatGifsService } from "./chat-gifs.service"; +import { ChatMediaController } from "./chat-media.controller"; +import { S3Module } from "src/s3/s3.module"; @Module({ imports: [ @@ -26,6 +32,7 @@ import { BackfillDirectMessages } from "./jobs/BackfillDirectMessages"; forwardRef(() => RconModule), NotificationsModule, PlayerBlocksModule, + S3Module, BullModule.registerQueue({ name: ChatQueues.ChatMaintenance, }), @@ -37,13 +44,17 @@ import { BackfillDirectMessages } from "./jobs/BackfillDirectMessages"; providers: [ ChatService, ChatGateway, + ChatAttachmentsService, + ChatGifsService, PruneDirectMessages, BackfillDirectMessages, + RemoveExpiredChatAttachments, + SweepChatAttachments, ...getQueuesProcessors("Chat"), loggerFactory(), ], exports: [ChatService], - controllers: [ChatController], + controllers: [ChatController, ChatMediaController], }) export class ChatModule implements OnModuleInit { constructor( @@ -79,5 +90,25 @@ export class ChatModule implements OnModuleInit { }, }, ); + + void this.maintenanceQueue.add( + RemoveExpiredChatAttachments.name, + {}, + { + repeat: { + pattern: "*/10 * * * *", + }, + }, + ); + + void this.maintenanceQueue.add( + SweepChatAttachments.name, + {}, + { + repeat: { + pattern: "41 4 * * *", + }, + }, + ); } } diff --git a/src/chat/chat.service.spec.ts b/src/chat/chat.service.spec.ts index 1d1524da..3f61d6ce 100644 --- a/src/chat/chat.service.spec.ts +++ b/src/chat/chat.service.spec.ts @@ -463,6 +463,13 @@ describe("ChatService direct messages", () => { { getConnection: () => redis } as any, push as any, playerBlocks as any, + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }); @@ -1086,7 +1093,13 @@ describe("ChatService direct messages", () => { service.sendMessageToChat(ChatLobbyType.Direct, room, player(), "hi"), ).resolves.toEqual({ accepted: true, messageId: expect.any(String) }); - expect(dmInserts().at(0)?.bindings.slice(1)).toEqual([room, ME, "hi"]); + expect(dmInserts().at(0)?.bindings.slice(1)).toEqual([ + room, + ME, + "hi", + null, + null, + ]); const incoming = redis.publish.mock.calls .map(([, payload]) => JSON.parse(payload)) @@ -1462,6 +1475,8 @@ describe("ChatService direct messages", () => { "2025-01-01T00:00:00.000Z", "web", ME, + null, + null, ]); }); @@ -2137,6 +2152,8 @@ describe("ChatService direct messages", () => { expect.any(String), "web", ME, + null, + null, ]); expect(push.retractChatMessage).toHaveBeenCalledWith(MESSAGE_ID); }); diff --git a/src/chat/chat.service.ts b/src/chat/chat.service.ts index 0c4c7ba2..643ca120 100644 --- a/src/chat/chat.service.ts +++ b/src/chat/chat.service.ts @@ -30,6 +30,19 @@ import { ChatReactions } from "./types/ChatReactions"; import { ChatReactResult } from "./types/ChatReactResult"; import { MatchChatArchiveEntry } from "./types/MatchChatArchiveEntry"; import { MatchChatLog } from "./types/MatchChatLog"; +import { ChatAttachment } from "./types/ChatAttachment"; +import { ChatGif } from "./types/ChatGif"; +import { + ChatAttachmentClaim, + ChatAttachmentRow, + ChatAttachmentsService, +} from "./chat-attachments.service"; +import { ChatGifsService } from "./chat-gifs.service"; + +export interface ChatMessageMedia { + attachments?: unknown; + gif?: unknown; +} @Injectable() export class ChatService { @@ -459,6 +472,8 @@ export class ChatService { private readonly redisManager: RedisManagerService, private readonly pushNotifications: PushNotificationsService, private readonly playerBlocks: PlayerBlocksService, + private readonly attachments: ChatAttachmentsService, + private readonly gifs: ChatGifsService, ) { this.redis = this.redisManager.getConnection(); } @@ -1030,16 +1045,22 @@ export class ChatService { // What a player typed on the website, or why it cannot be sent. Lines relayed // from the game are not held to this: the game has already limited them. + // A message carrying files or a GIF needs no text. public static messageText( raw: unknown, + allowEmpty = false, ): { text: string } | { error: ChatErrorCode } { + if (allowEmpty && (raw === undefined || raw === null)) { + return { text: "" }; + } + if (typeof raw !== "string") { return { error: ChatErrorCode.Invalid }; } const text = raw.trim(); - if (text.length === 0) { + if (text.length === 0 && !allowEmpty) { return { error: ChatErrorCode.Invalid }; } @@ -1060,6 +1081,72 @@ export class ChatService { ); } + public static hasMedia(media?: ChatMessageMedia): boolean { + return ( + (Array.isArray(media?.attachments) && media.attachments.length > 0) || + (media?.gif !== undefined && media?.gif !== null) + ); + } + + // The ids of files the composer uploaded, or a GIPHY GIF -- never both, and + // never more than a message holds. + public static messageMedia( + media?: ChatMessageMedia, + ): + | { attachmentIds: string[]; gif: ChatGif | null } + | { error: ChatErrorCode } { + const raw = media?.attachments ?? []; + const hasGif = media?.gif !== undefined && media?.gif !== null; + + if (!Array.isArray(raw)) { + return { error: ChatErrorCode.Invalid }; + } + + if ( + raw.length > ChatAttachmentsService.MAX_PER_MESSAGE || + raw.some((id) => typeof id !== "string" || !ChatService.UUID.test(id)) + ) { + return { error: ChatErrorCode.Invalid }; + } + + // Postgres hands ids back lowercase, and the claim matches on them. + const ids = (raw as string[]).map((id) => id.toLowerCase()); + + if (new Set(ids).size !== ids.length) { + return { error: ChatErrorCode.Invalid }; + } + + if (!hasGif) { + return { attachmentIds: ids, gif: null }; + } + + const gif = ChatGifsService.gif(media.gif); + + if (!gif || raw.length > 0) { + return { error: ChatErrorCode.Invalid }; + } + + return { attachmentIds: [], gif }; + } + + // What a push says for a message: its text, or what it carries when there + // is none. + public static previewText( + text: string, + attachments: number, + gif: boolean, + ): string { + if (text) { + return text; + } + + if (gif) { + return "GIF"; + } + + return attachments === 1 ? "Attachment" : `${attachments} attachments`; + } + public async sendMessageToChat( type: ChatLobbyType, id: string, @@ -1067,11 +1154,26 @@ export class ChatService { _message: string, skipCheck = false, source: ChatMessageSource = "web", + _media?: ChatMessageMedia, ): Promise { let text = _message; + let attachmentIds: string[] = []; + let gif: ChatGif | null = null; if (source === "web") { - const parsed = ChatService.messageText(_message); + const media = ChatService.messageMedia(_media); + + if ("error" in media) { + return { accepted: false, code: media.error }; + } + + attachmentIds = media.attachmentIds; + gif = media.gif; + + const parsed = ChatService.messageText( + _message, + attachmentIds.length > 0 || gif !== null, + ); if ("error" in parsed) { return { accepted: false, code: parsed.error }; @@ -1090,6 +1192,12 @@ export class ChatService { text = parsed.text; } + const hasMedia = attachmentIds.length > 0 || gif !== null; + + if (hasMedia && !ChatAttachmentsService.allowsAttachments(type)) { + return { accepted: false, code: ChatErrorCode.NotAllowed }; + } + if (skipCheck === false && !(await this.canPostIn(type, id, player))) { return { accepted: false, code: ChatErrorCode.NotAllowed }; } @@ -1105,6 +1213,10 @@ export class ChatService { return { accepted: false, code: ChatErrorCode.Gagged }; } + if (gif && !(await this.gifs.enabled())) { + return { accepted: false, code: ChatErrorCode.NotAllowed }; + } + const name = await this.redis.get( HasuraService.PLAYER_NAME_CACHE_KEY(player.steam_id), ); @@ -1128,28 +1240,110 @@ export class ChatService { avatar_url: player.avatar_url, profile_url: player.profile_url, }, + ...(gif ? { gif } : {}), + }; + + const claim = { + type, + roomId: id, + steamId: String(player.steam_id), + messageId: message.id, + expiresAt: ChatAttachmentsService.expiresOnSend( + type, + this.ttlFor(type), + timestamp, + ), }; if (type === ChatLobbyType.Direct) { - if (!(await this.storeDirectMessage(id, message))) { - return { accepted: false, code: ChatErrorCode.NotAllowed }; + const refusal = await this.storeDirectMessage( + id, + message, + attachmentIds, + claim, + ); + + if (refusal) { + return { + accepted: false, + code: await this.claimRefusal(refusal, attachmentIds, claim), + }; } } else { const messageKey = `chat_${type}_${id}`; - // Keyed by id and not `${steam_id}:${now}`, which silently dropped a - // message when the same player landed two within the same millisecond. - const messageField = message.id; - await this.redis.hset(messageKey, messageField, JSON.stringify(message)); - await this.redis.sendCommand( - new Redis.Command("HEXPIRE", [ + const store = async () => { + // Keyed by id and not `${steam_id}:${now}`, which silently dropped a + // message when the same player landed two within the same millisecond. + const messageField = message.id; + await this.redis.hset( messageKey, - this.ttlFor(type), - "FIELDS", - 1, messageField, - ]), - ); + JSON.stringify(message), + ); + + await this.redis.sendCommand( + new Redis.Command("HEXPIRE", [ + messageKey, + this.ttlFor(type), + "FIELDS", + 1, + messageField, + ]), + ); + }; + + if (attachmentIds.length > 0) { + // The claim commits only once the message is stored, so a write that + // fails gives the files back rather than binding them to nothing. The + // message goes in first, so whatever fails after it -- its expiry, the + // commit -- takes it back out again. + let written = false; + let refused: boolean; + + try { + refused = await this.postgres.transaction(async (client) => { + const claimed = await this.attachments.claim( + attachmentIds, + claim, + client, + ); + + if (!claimed) { + return true; + } + + message.attachments = claimed; + written = true; + await store(); + + return false; + }); + } catch (error) { + if (written) { + await this.redis.hdel(messageKey, message.id).catch(() => { + this.logger.warn( + `unable to take back ${type}:${id} message ${message.id}`, + ); + }); + } + + throw error; + } + + if (refused) { + return { + accepted: false, + code: await this.claimRefusal( + ChatErrorCode.Invalid, + attachmentIds, + claim, + ), + }; + } + } else { + await store(); + } } const outgoing: ChatMessage = { ...message, reactions: {} }; @@ -1175,7 +1369,11 @@ export class ChatService { id, player, message.from.name, - text, + ChatService.previewText( + text, + message.attachments?.length ?? 0, + !!message.gif, + ), message.id, ).catch((error) => { this.logger.warn(`unable to notify ${type}:${id} of a message`, error); @@ -1220,6 +1418,125 @@ export class ChatService { private static readonly UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + private static readonly ATTACHMENT_ACCESS_TTL_SECONDS = 60; + + // Whether a player may start uploading into a room: the same rule as + // sending to it, so nothing is uploaded that could never be sent. + public async attachmentRefusal( + type: ChatLobbyType, + id: string, + user: User, + ): Promise { + if (!ChatAttachmentsService.allowsAttachments(type)) { + return ChatErrorCode.NotAllowed; + } + + const current = await this.getCurrentUser(String(user.steam_id)); + + if (!current || !(await this.canPostIn(type, id, current))) { + return ChatErrorCode.NotAllowed; + } + + if ( + type !== ChatLobbyType.Direct && + (await this.isGagged(current.steam_id)) + ) { + return ChatErrorCode.Gagged; + } + + return null; + } + + // A send whose answer never arrived is retried with the same files. The + // first one landed, so the retry is told so rather than that it failed. + private async claimRefusal( + refusal: ChatErrorCode, + attachmentIds: string[], + claim: ChatAttachmentClaim, + ): Promise { + if ( + refusal === ChatErrorCode.Invalid && + attachmentIds.length > 0 && + (await this.attachments.sentBy(attachmentIds, claim)) + ) { + return ChatErrorCode.AlreadySent; + } + + return refusal; + } + + // Judged on who the player is now, not on the role their session was + // signed in with. + public async canViewAttachment( + row: ChatAttachmentRow, + sessionUser: User | undefined, + ): Promise { + if (!sessionUser?.steam_id) { + return false; + } + + const current = await this.attachmentViewer(String(sessionUser.steam_id)); + + if (!current) { + return false; + } + + return await ChatAttachmentsService.canView(row, current, () => + this.canViewRoom(row.room_type, row.room_id, current), + ); + } + + // The row is read fresh on every request, so a deleted file goes dark at + // once; who the viewer is only changes with their role, and is kept for the + // same minute as their room access. + private async attachmentViewer(steamId: string): Promise { + const cacheKey = `chat:attachment-viewer:${steamId}`; + const cached = await this.redis.get(cacheKey); + + if (cached !== null) { + return JSON.parse(cached) as User; + } + + const current = await this.getCurrentUser(steamId); + + if (current) { + await this.redis.set( + cacheKey, + JSON.stringify(current), + "EX", + ChatService.ATTACHMENT_ACCESS_TTL_SECONDS, + ); + } + + return current; + } + + // Asked for every image and every range of a video, so the answer is kept + // for a minute rather than going back to hasura each time. + private async canViewRoom( + type: ChatLobbyType, + id: string, + user: User, + ): Promise { + const cacheKey = `chat:attachment-access:${user.steam_id}:${type}:${id}`; + const cached = await this.redis.get(cacheKey); + + if (cached !== null) { + return cached === "1"; + } + + const allowed = await this.canAccessLobby(type, id, user); + + await this.redis.set( + cacheKey, + allowed ? "1" : "0", + "EX", + ChatService.ATTACHMENT_ACCESS_TTL_SECONDS, + ); + + return allowed; + } + public async deleteMessage( type: ChatLobbyType, id: string, @@ -1289,9 +1606,32 @@ export class ChatService { await this.retractNotifications(type, id, messageId); + await this.removeAttachments(type, id, messageId); + return { deleted: true }; } + // A group room's files stay as evidence until they expire, hidden from the + // room; a direct message's go at once. Never allowed to fail the delete: a + // file left behind still expires, and the sweep takes it then. + private async removeAttachments( + type: ChatLobbyType, + id: string, + messageId: string, + ) { + const removal = + type === ChatLobbyType.Direct + ? this.attachments.expireMessage(type, id, messageId) + : this.attachments.markDeleted(type, id, messageId); + + await removal.catch((error) => { + this.logger.warn( + `unable to remove the files of ${type}:${id} message ${messageId}`, + error, + ); + }); + } + private async deleteRefusal( message: ChatMessage, type: ChatLobbyType, @@ -1757,6 +2097,8 @@ export class ChatService { await this.retractNotifications(ChatLobbyType.Direct, roomId, messageId); + await this.removeAttachments(ChatLobbyType.Direct, roomId, messageId); + return { deleted: true }; } @@ -1865,11 +2207,12 @@ export class ChatService { await this.postgres.query( `INSERT INTO public.chat_message_deletions (message_id, room_type, room_id, author_steam_id, message, - message_created_at, source, deleted_by_steam_id) + message_created_at, source, deleted_by_steam_id, + attachments, gif) SELECT $1::uuid, $2, $3, (SELECT steam_id FROM public.players WHERE steam_id = $4::bigint), - $5, $6::timestamptz, $7, $8::bigint + $5, $6::timestamptz, $7, $8::bigint, $9::jsonb, $10::jsonb ON CONFLICT (room_type, room_id, message_id) DO NOTHING`, [ messageId, @@ -1880,6 +2223,10 @@ export class ChatService { ChatService.messageCreatedAt(message), message.source ?? null, deletedBy.steam_id, + message.attachments?.length + ? JSON.stringify(message.attachments) + : null, + message.gif ? JSON.stringify(message.gif) : null, ], ); } @@ -2403,28 +2750,77 @@ export class ChatService { // // A block committed after the send's access check still stops the insert, // and with it the rail, the delivery and the notification. + // + // Its files are claimed in the same transaction, so a refused insert gives + // them back. private async storeDirectMessage( roomId: string, - message: { id: string; message: string; from: User }, - ): Promise { + message: ChatMessage, + attachmentIds: string[], + claim: ChatAttachmentClaim, + ): Promise { const parties = parseDirectRoomId(roomId); if (!parties) { - return false; + return ChatErrorCode.NotAllowed; } - const stored = await this.postgres.query>( - `INSERT INTO public.direct_messages (id, room_id, from_steam_id, message) - SELECT $1::uuid, $2, $3::bigint, $4 - WHERE NOT public.is_blocked_either_way( - split_part($2, ':', 1)::bigint, - split_part($2, ':', 2)::bigint) - RETURNING id::text AS id`, - [message.id, roomId, message.from.steam_id, message.message], - ); + try { + const refusal = await this.postgres.transaction(async (client) => { + let attachments: ChatAttachment[] = []; + + if (attachmentIds.length > 0) { + const claimed = await this.attachments.claim( + attachmentIds, + claim, + client, + ); - if (stored.length === 0) { - return false; + if (!claimed) { + return ChatErrorCode.Invalid; + } + + attachments = claimed; + } + + const { rows } = await client.query( + `INSERT INTO public.direct_messages + (id, room_id, from_steam_id, message, attachments, gif) + SELECT $1::uuid, $2, $3::bigint, $4, $5::jsonb, $6::jsonb + WHERE NOT public.is_blocked_either_way( + split_part($2, ':', 1)::bigint, + split_part($2, ':', 2)::bigint) + RETURNING id::text AS id`, + [ + message.id, + roomId, + message.from.steam_id, + message.message, + attachments.length > 0 ? JSON.stringify(attachments) : null, + message.gif ? JSON.stringify(message.gif) : null, + ], + ); + + if (rows.length === 0) { + throw ChatService.DIRECT_MESSAGE_REFUSED; + } + + if (attachments.length > 0) { + message.attachments = attachments; + } + + return null; + }); + + if (refusal) { + return refusal; + } + } catch (error) { + if (error === ChatService.DIRECT_MESSAGE_REFUSED) { + return ChatErrorCode.NotAllowed; + } + + throw error; } // A message puts the conversation back on the bar, even if it was removed @@ -2452,9 +2848,14 @@ export class ChatService { await this.enforceDirectBarLimit(parties); - return true; + return null; } + // Thrown to roll back a claim whose message the block check refused. + private static readonly DIRECT_MESSAGE_REFUSED = new Error( + "direct message refused", + ); + // How many conversations the rail holds. Past this the quietest one drops // off -- it still exists, and comes back the moment that person writes. private static readonly MAX_DIRECT_TABS = 8; @@ -2550,6 +2951,8 @@ export class ChatService { created_at: Date; edited_at: Date | null; reactions: ChatReactions | null; + attachments: ChatAttachment[] | null; + gif: ChatGif | null; steam_id: string; name: string; role: e_player_roles_enum; @@ -2558,7 +2961,7 @@ export class ChatService { }> >( `SELECT dm.id::text AS id, dm.message, dm.created_at, dm.edited_at, - reactions.reactions, + dm.attachments, dm.gif, reactions.reactions, p.steam_id::text AS steam_id, p.name, p.role::text AS role, p.avatar_url, p.profile_url FROM public.direct_messages dm @@ -2581,6 +2984,8 @@ export class ChatService { ? { edited_at: new Date(row.edited_at).toISOString() } : {}), reactions: ChatService.orderedReactions(row.reactions), + ...(row.attachments ? { attachments: row.attachments } : {}), + ...(row.gif ? { gif: row.gif } : {}), from: { role: row.role, name: row.name, @@ -3438,6 +3843,25 @@ export class ChatService { await this.removeLobby(fromType, fromId); + await this.attachments + .moveRoom( + fromType, + fromId, + toType, + toId, + ChatAttachmentsService.expiresOnSend( + toType, + this.ttlFor(toType), + new Date(), + ), + ) + .catch((error) => { + this.logger.warn( + `unable to move the files of ${fromType}:${fromId} to ${toType}:${toId}`, + error, + ); + }); + if (!Array.isArray(moved) || moved.length === 0) { return; } diff --git a/src/chat/enums/ChatErrorCode.ts b/src/chat/enums/ChatErrorCode.ts index 38814cf9..69424593 100644 --- a/src/chat/enums/ChatErrorCode.ts +++ b/src/chat/enums/ChatErrorCode.ts @@ -8,4 +8,12 @@ export enum ChatErrorCode { NotFound = "not_found", WindowClosed = "window_closed", RateLimited = "rate_limited", + TooLarge = "too_large", + UnsupportedType = "unsupported_type", + TooManyPending = "too_many_pending", + Disabled = "disabled", + Unavailable = "unavailable", + QuotaExceeded = "quota_exceeded", + Busy = "busy", + AlreadySent = "already_sent", } diff --git a/src/chat/jobs/PruneDirectMessages.ts b/src/chat/jobs/PruneDirectMessages.ts index 9cee2057..0a114685 100644 --- a/src/chat/jobs/PruneDirectMessages.ts +++ b/src/chat/jobs/PruneDirectMessages.ts @@ -5,6 +5,7 @@ import { UseQueue } from "../../utilities/QueueProcessors"; import { ChatQueues } from "../enums/ChatQueues"; import { PostgresService } from "../../postgres/postgres.service"; import { SystemSettingName } from "../../system/enums/SystemSettingName"; +import { ChatAttachmentsService } from "../chat-attachments.service"; const DEFAULT_RETENTION_DAYS = 365; @@ -20,6 +21,7 @@ export class PruneDirectMessages extends WorkerHost { constructor( private readonly logger: Logger, private readonly postgres: PostgresService, + private readonly attachments: ChatAttachmentsService, ) { super(); } @@ -54,6 +56,10 @@ export class PruneDirectMessages extends WorkerHost { `pruned ${deleted.length} direct message(s) older than ${days} days`, ); } + + // Deleting a message marks its files due (hasura/triggers/direct_messages), + // so they go in the same run rather than at the next sweep. + await this.attachments.removeExpired(); } private async retentionDays(): Promise { diff --git a/src/chat/jobs/RemoveExpiredChatAttachments.ts b/src/chat/jobs/RemoveExpiredChatAttachments.ts new file mode 100644 index 00000000..e9ce0e73 --- /dev/null +++ b/src/chat/jobs/RemoveExpiredChatAttachments.ts @@ -0,0 +1,17 @@ +import { Job } from "bullmq"; +import { WorkerHost } from "@nestjs/bullmq"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { ChatQueues } from "../enums/ChatQueues"; +import { ChatAttachmentsService } from "../chat-attachments.service"; + +// Room files whose room has let go of their message, and uploads never sent. +@UseQueue("Chat", ChatQueues.ChatMaintenance) +export class RemoveExpiredChatAttachments extends WorkerHost { + constructor(private readonly attachments: ChatAttachmentsService) { + super(); + } + + async process(_job: Job): Promise { + return await this.attachments.removeExpired(); + } +} diff --git a/src/chat/jobs/SweepChatAttachments.ts b/src/chat/jobs/SweepChatAttachments.ts new file mode 100644 index 00000000..49556978 --- /dev/null +++ b/src/chat/jobs/SweepChatAttachments.ts @@ -0,0 +1,18 @@ +import { Job } from "bullmq"; +import { WorkerHost } from "@nestjs/bullmq"; +import { UseQueue } from "../../utilities/QueueProcessors"; +import { ChatQueues } from "../enums/ChatQueues"; +import { ChatAttachmentsService } from "../chat-attachments.service"; + +@UseQueue("Chat", ChatQueues.ChatMaintenance) +export class SweepChatAttachments extends WorkerHost { + constructor(private readonly attachments: ChatAttachmentsService) { + super(); + } + + async process(_job: Job): Promise { + await this.attachments.removeExpired(); + + return await this.attachments.sweepOrphans(); + } +} diff --git a/src/chat/tournament-chat-retention.spec.ts b/src/chat/tournament-chat-retention.spec.ts index 3fb46b35..ab12c582 100644 --- a/src/chat/tournament-chat-retention.spec.ts +++ b/src/chat/tournament-chat-retention.spec.ts @@ -34,6 +34,13 @@ describe("tournament chat retention", () => { { getConnection: () => redis } as any, { sendChatMessage: jest.fn() } as any, { blockedAmong: jest.fn(async () => new Map()) } as any, + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); await service.sendMessageToChat( diff --git a/src/chat/types/ChatAttachment.ts b/src/chat/types/ChatAttachment.ts new file mode 100644 index 00000000..67eb7c48 --- /dev/null +++ b/src/chat/types/ChatAttachment.ts @@ -0,0 +1,15 @@ +export type ChatAttachmentKind = "image" | "video"; + +// What a message carries for each file. The file itself is served by id from +// /chat/attachments, behind the same access check as its room. +export interface ChatAttachment { + id: string; + kind: ChatAttachmentKind; + name: string; + mime_type: string; + size: number; + width?: number; + height?: number; + duration_ms?: number; + poster?: boolean; +} diff --git a/src/chat/types/ChatGif.ts b/src/chat/types/ChatGif.ts new file mode 100644 index 00000000..8c71b05f --- /dev/null +++ b/src/chat/types/ChatGif.ts @@ -0,0 +1,7 @@ +// A GIPHY GIF by id. The web builds the media URL from the id, so a message can +// never carry an arbitrary URL dressed up as a GIF. +export interface ChatGif { + id: string; + width: number; + height: number; +} diff --git a/src/chat/types/ChatMessage.ts b/src/chat/types/ChatMessage.ts index b18b11b1..e44dd3ae 100644 --- a/src/chat/types/ChatMessage.ts +++ b/src/chat/types/ChatMessage.ts @@ -1,5 +1,7 @@ import { e_player_roles_enum } from "generated"; import { ChatReactions } from "./ChatReactions"; +import { ChatAttachment } from "./ChatAttachment"; +import { ChatGif } from "./ChatGif"; export type ChatMessageSource = "web" | "game"; @@ -14,6 +16,8 @@ export interface ChatMessage { // Added when a message is sent to clients, never stored in its JSON: an // edit's compare-and-set would otherwise contend with every reaction. reactions?: ChatReactions; + attachments?: ChatAttachment[]; + gif?: ChatGif; from: { role: e_player_roles_enum; name: string; diff --git a/src/hasura/metadata-permissions.spec.ts b/src/hasura/metadata-permissions.spec.ts index 6b6b01c4..94c88270 100644 --- a/src/hasura/metadata-permissions.spec.ts +++ b/src/hasura/metadata-permissions.spec.ts @@ -151,6 +151,51 @@ describe("hasura table metadata", () => { // A warning is a private note between the player and staff; every other // sanction stays on the public record. + // A secret setting is kept from administrators by its name alone, so a role + // that could rename the row could move it out of the list and read it. + describe("secret settings", () => { + const SECRETS = ["web_push_private_key", "giphy_api_key"]; + + const settings = () => + tables.find(({ file }) => file === "public_settings.yaml")?.metadata; + + it("never shows a secret to an administrator", () => { + const hidden = + blocksByRole(settings(), "select_permissions").get("administrator") + ?.filter?.name?._nin ?? []; + + expect(hidden).toEqual(expect.arrayContaining(SECRETS)); + }); + + it("never lets any role rename a setting", () => { + const renamers = (settings()?.update_permissions ?? []) + .filter((entry: Record) => + (entry.permission?.columns ?? []).includes("name"), + ) + .map((entry: { role: string }) => entry.role); + + expect(renamers).toEqual([]); + }); + }); + + describe("chat message deletions", () => { + it("shows staff the files and GIF a deleted message carried", () => { + const deletions = tables.find( + ({ file }) => file === "public_chat_message_deletions.yaml", + )?.metadata; + + for (const [role, permission] of blocksByRole( + deletions, + "select_permissions", + )) { + expect([role, permission.columns]).toEqual([ + role, + expect.arrayContaining(["attachments", "gif"]), + ]); + } + }); + }); + describe("player_sanctions warnings", () => { const sanctions = () => tables.find(({ file }) => file === "public_player_sanctions.yaml") diff --git a/src/s3/s3.service.spec.ts b/src/s3/s3.service.spec.ts index 04f7d974..a05024de 100644 --- a/src/s3/s3.service.spec.ts +++ b/src/s3/s3.service.spec.ts @@ -134,3 +134,67 @@ describe("S3Service presigned url routing", () => { }, ); }); + +describe("S3Service.removePrefix", () => { + const withClient = (deleted: Record) => { + const service = build("rustfs", "9000", false); + const send = jest.fn(async (command: { constructor: { name: string } }) => + command.constructor.name === "ListObjectVersionsCommand" + ? { + Versions: [ + { + Key: "chat-attachments/rooms/2026-10-01/a-1/file", + VersionId: "v1", + }, + ], + IsTruncated: false, + } + : deleted, + ); + + (service as any).rawClients.set("internal", { send, destroy: jest.fn() }); + + return { service, send }; + }; + + it("counts what it removed", async () => { + const { service } = withClient({}); + + await expect( + service.removePrefix("chat-attachments/rooms/2026-10-01/a-1/"), + ).resolves.toBe(1); + }); + + const refused = { + Errors: [ + { + Key: "chat-attachments/rooms/2026-10-01/a-1/file", + Code: "AccessDenied", + Message: "no deleteFiles capability", + }, + ], + }; + + // Match, clip and event deletes have always carried on past a key the store + // refused; they log it rather than fail half way through. + it("logs a key the store refused, and carries on", async () => { + const { service } = withClient(refused); + const warn = jest.spyOn((service as any).logger, "warn"); + + await expect( + service.removePrefix("chat-attachments/rooms/2026-10-01/a-1/"), + ).resolves.toBe(1); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/AccessDenied/)); + }); + + // DeleteObjects answers 200 even when it removed nothing. A caller whose + // row is the only record of the file has to know, or it forgets a file that + // is still being billed. + it("fails a strict sweep when the store refused any of it", async () => { + const { service } = withClient(refused); + + await expect( + service.removePrefixStrictly("chat-attachments/rooms/2026-10-01/a-1/"), + ).rejects.toThrow(/AccessDenied/); + }); +}); diff --git a/src/s3/s3.service.ts b/src/s3/s3.service.ts index 9465a386..4e109055 100644 --- a/src/s3/s3.service.ts +++ b/src/s3/s3.service.ts @@ -8,6 +8,7 @@ import { CompleteMultipartUploadCommand, CreateMultipartUploadCommand, DeleteObjectsCommand, + ListObjectsV2Command, ListObjectVersionsCommand, ListPartsCommand, S3Client, @@ -49,6 +50,8 @@ const importESM = new Function("specifier", "return import(specifier)") as ( @Injectable() export class S3Service implements OnModuleDestroy { + public static PART_IDLE_TIMEOUT_MS = 30_000; + private bucket: string; private config: S3Config; private modules?: Promise; @@ -341,6 +344,24 @@ export class S3Service implements OnModuleDestroy { public async removePrefix( prefix: string, bucket: string = this.bucket, + ): Promise { + return await this.sweepPrefix(prefix, bucket, false); + } + + // DeleteObjects refuses keys inside a 200, so only a strict sweep notices. + // For a caller whose own row is the only record of the files: it must keep + // that row and try again rather than forget objects still being billed. + public async removePrefixStrictly( + prefix: string, + bucket: string = this.bucket, + ): Promise { + return await this.sweepPrefix(prefix, bucket, true); + } + + private async sweepPrefix( + prefix: string, + bucket: string, + strict: boolean, ): Promise { const client = this.raw(); @@ -375,12 +396,26 @@ export class S3Service implements OnModuleDestroy { for (let i = 0; i < entries.length; i += 1000) { const batch = entries.slice(i, i + 1000); - await client.send( + const result = await client.send( new DeleteObjectsCommand({ Bucket: bucket, Delete: { Objects: batch }, }), ); + + const [failure] = result?.Errors ?? []; + + if (failure) { + const message = + `unable to remove ${result.Errors.length} object(s) under ${prefix}: ${failure.Code} ${failure.Message ?? ""}`.trim(); + + if (strict) { + throw new Error(message); + } + + this.logger.warn(message); + } + removed += batch.length; } @@ -393,6 +428,38 @@ export class S3Service implements OnModuleDestroy { return removed; } + // The "folders" directly under a prefix, without listing what is in them. + public async listPrefixes( + prefix: string, + bucket: string = this.bucket, + ): Promise { + const client = this.raw(); + + const prefixes: string[] = []; + let token: string | undefined; + + do { + const listed = await client.send( + new ListObjectsV2Command({ + Bucket: bucket, + Prefix: prefix, + Delimiter: "/", + ContinuationToken: token, + }), + ); + + for (const common of listed.CommonPrefixes ?? []) { + if (common.Prefix) { + prefixes.push(common.Prefix); + } + } + + token = listed.IsTruncated ? listed.NextContinuationToken : undefined; + } while (token); + + return prefixes; + } + public async removeKeys( keys: string[], bucket: string = this.bucket, @@ -502,6 +569,60 @@ export class S3Service implements OnModuleDestroy { ); } + // Streamed through with its length rather than buffered. Checksums only + // where an operation requires one: by default the SDK would wrap a stream + // in an aws-chunked body with a trailing checksum, which not every + // S3-compatible store accepts. + public async uploadPart( + key: string, + uploadId: string, + partNumber: number, + body: Readable | Buffer, + length: number, + abortSignal?: AbortSignal, + bucket: string = this.bucket, + ): Promise { + await this.streamingClient().send( + new UploadPartCommand({ + Bucket: bucket, + Key: key, + UploadId: uploadId, + PartNumber: partNumber, + Body: body, + ContentLength: length, + }), + { abortSignal }, + ); + } + + private streamingClient(): S3Client { + const cached = this.rawClients.get("streaming"); + if (cached) { + return cached; + } + + const client = new S3Client({ + endpoint: this.endpointFor(false), + region: this.config.region, + forcePathStyle: this.forcePathStyle, + credentials: this.credentials, + requestChecksumCalculation: "WHEN_REQUIRED", + // A streamed body cannot be sent twice. + maxAttempts: 1, + // Idle, not total: a slow sender is fine, a store that stops answering + // is not -- it holds a player's upload slot and a socket on both ends. + // requestTimeout would only log a warning here. + requestHandler: { + connectionTimeout: 10_000, + socketTimeout: S3Service.PART_IDLE_TIMEOUT_MS, + }, + }); + + this.rawClients.set("streaming", client); + + return client; + } + public async completeMultipartUpload( key: string, uploadId: string, diff --git a/src/system/enums/SystemSettingName.ts b/src/system/enums/SystemSettingName.ts index 1cdec93f..52b0f27c 100644 --- a/src/system/enums/SystemSettingName.ts +++ b/src/system/enums/SystemSettingName.ts @@ -13,6 +13,14 @@ export enum SystemSettingName { // Days, not seconds: DMs are swept rather than expired, because they live in // postgres. 0 keeps them forever. ChatRetentionDirectDays = "public.chat_retention_direct_days", + ChatAttachmentMaxMb = "chat_attachment_max_mb", + ChatAttachmentDailyMb = "chat_attachment_daily_mb", + // Calls to GIPHY per hour across the whole panel: the key is one quota. + GiphyHourlyLimit = "giphy_hourly_limit", + // Write-only for administrators: public_settings.yaml leaves it out of their + // select permission, and searches go through the api so it never reaches a + // browser. + GiphyApiKey = "giphy_api_key", DemoNetworkLimiter = "demo_network_limiter", PublicDefaultModels = "public.default_models", VetoPickTimeout = "public.veto_pick_timeout", diff --git a/test/chat-attachments-migration.spec.ts b/test/chat-attachments-migration.spec.ts new file mode 100644 index 00000000..a7b69a71 --- /dev/null +++ b/test/chat-attachments-migration.spec.ts @@ -0,0 +1,71 @@ +import { readFileSync } from "fs"; +import { join, resolve } from "path"; +import { bootMigratedDb, SqlTestDb } from "./utils/sql-test-db"; + +const MIGRATIONS = resolve(__dirname, "../hasura/migrations/default"); + +// 1890000000200 shipped before the deletion evidence and the upload ledger +// existed, and a stack that ran it will never run it again. Whatever came +// after has to arrive under a version of its own. +describe("chat attachments migrations", () => { + let db: SqlTestDb; + + beforeAll(async () => { + db = await bootMigratedDb("ChatAttachmentsMigrationTest"); + }, 600_000); + + afterAll(async () => { + await db?.stop(); + }); + + const columns = async (table: string) => + ( + await db.postgres.query>( + `SELECT column_name FROM information_schema.columns + WHERE table_schema = 'public' AND table_name = $1`, + [table], + ) + ).map(({ column_name }) => column_name); + + it("leaves the migration that shipped as it shipped", () => { + const shipped = readFileSync( + join(MIGRATIONS, "1890000000200_chat_attachments/up.sql"), + "utf8", + ); + + expect(shipped).not.toMatch(/chat_attachment_usage|deleted_at/); + }); + + it("brings a stack that ran only the shipped migration up to date", async () => { + await db.postgres.query(` + DROP TABLE IF EXISTS public.chat_attachment_usage; + ALTER TABLE public.chat_attachments DROP COLUMN IF EXISTS deleted_at; + ALTER TABLE public.chat_message_deletions + DROP COLUMN IF EXISTS attachments, + DROP COLUMN IF EXISTS gif; + DELETE FROM hdb_catalog.schema_migrations + WHERE version > 1890000000200 AND version < 1890000000300; + `); + + await (db.hasura as any).applyMigrations(MIGRATIONS); + + expect(await columns("chat_attachments")).toContain("deleted_at"); + expect(await columns("chat_attachment_usage")).toEqual( + expect.arrayContaining(["steam_id", "bytes", "created_at"]), + ); + expect(await columns("chat_message_deletions")).toEqual( + expect.arrayContaining(["attachments", "gif"]), + ); + }); + + it("runs the follow-up again without complaint", async () => { + await db.postgres.query(` + DELETE FROM hdb_catalog.schema_migrations + WHERE version > 1890000000200 AND version < 1890000000300; + `); + + await expect( + (db.hasura as any).applyMigrations(MIGRATIONS), + ).resolves.toEqual(expect.any(Number)); + }); +}); diff --git a/test/chat-attachments.spec.ts b/test/chat-attachments.spec.ts new file mode 100644 index 00000000..2044654f --- /dev/null +++ b/test/chat-attachments.spec.ts @@ -0,0 +1,840 @@ +import { randomUUID } from "crypto"; +import { Readable } from "stream"; +import { PostgresService } from "./../src/postgres/postgres.service"; +import { Fixtures } from "./utils/fixtures"; +import { bootMigratedDb, SqlTestDb } from "./utils/sql-test-db"; +import { ChatService } from "./../src/chat/chat.service"; +import { ChatAttachmentsService } from "./../src/chat/chat-attachments.service"; +import { ChatGifsService } from "./../src/chat/chat-gifs.service"; +import { PlayerBlocksService } from "./../src/player-blocks/player-blocks.service"; +import { ChatErrorCode } from "./../src/chat/enums/ChatErrorCode"; +import { ChatLobbyType } from "./../src/chat/enums/ChatLobbyTypes"; +import { PruneDirectMessages } from "./../src/chat/jobs/PruneDirectMessages"; +import { directRoomId } from "./../src/chat/utilities/directRoomId"; + +const uint32 = (value: number) => { + const bytes = Buffer.alloc(4); + bytes.writeUInt32BE(value); + return bytes; +}; + +const PNG = Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + uint32(13), + Buffer.from("IHDR"), + uint32(640), + uint32(360), + Buffer.alloc(40), +]); + +describe("chat attachments (SQL-driven)", () => { + let db: SqlTestDb; + let postgres: PostgresService; + let fx: Fixtures; + let chat: ChatService; + let attachments: ChatAttachmentsService; + + const logger = { log: jest.fn(), warn: jest.fn(), error: jest.fn() }; + + // What the bucket holds, by key, for whatever uploads and sweeps touch it. + let objects: Map; + + const s3 = { + createMultipartUpload: jest.fn(async () => `upload-${randomUUID()}`), + uploadPart: jest.fn( + async (key: string, _uploadId: string, _part: number, body: Readable) => { + let length = 0; + for await (const chunk of body) { + length += (chunk as Buffer).length; + } + objects.set(key, (objects.get(key) ?? 0) + length); + }, + ), + completeMultipartUpload: jest.fn(async () => {}), + abortMultipartUpload: jest.fn(async () => {}), + stat: jest.fn(async (key: string) => ({ + size: objects.get(key) ?? 0, + metaData: {}, + })), + put: jest.fn(async (key: string, body: Buffer) => { + objects.set(key, body.length); + }), + removePrefixStrictly: jest.fn(async (prefix: string) => { + let removed = 0; + for (const key of [...objects.keys()]) { + if (key.startsWith(prefix)) { + objects.delete(key); + removed++; + } + } + return removed; + }), + listPrefixes: jest.fn(async (): Promise => []), + listStream: jest.fn(async function* () {}), + }; + + const push = { + sendChatMessage: jest.fn(async () => {}), + retractChatMessage: jest.fn(async () => {}), + editChatMessage: jest.fn(async () => {}), + }; + + const redis = { + hset: jest.fn(), + hget: jest.fn().mockResolvedValue(null), + hgetall: jest.fn().mockResolvedValue({}), + hdel: jest.fn().mockResolvedValue(1), + get: jest.fn().mockResolvedValue(null), + set: jest.fn(), + del: jest.fn(), + expire: jest.fn(), + publish: jest.fn(), + sendCommand: jest.fn(), + eval: jest.fn(async (script: string) => + script.includes("INCR") ? 1 : [1, 1], + ), + }; + + beforeAll(async () => { + db = await bootMigratedDb("ChatAttachmentsTest"); + postgres = db.postgres; + fx = new Fixtures(postgres, 76561199500000000n); + + attachments = new ChatAttachmentsService( + logger as any, + postgres, + s3 as any, + ); + + chat = new ChatService( + logger as any, + {} as any, + { + query: jest.fn(async (query: Record) => { + if (query.players_by_pk) { + return { + players_by_pk: { + steam_id: query.players_by_pk.__args.steam_id, + name: "Someone", + role: "moderator", + }, + }; + } + + if (query.lobby_players_by_pk) { + return { lobby_players_by_pk: { status: "Accepted" } }; + } + + return { friends: [{ status: "Accepted" }] }; + }), + } as any, + postgres, + { getConnection: () => redis } as any, + push as any, + new PlayerBlocksService(postgres), + attachments, + new ChatGifsService(logger as any, postgres, { + getConnection: () => redis, + } as any), + ); + }, 600_000); + + afterAll(async () => { + await db?.stop(); + }); + + beforeEach(async () => { + jest.clearAllMocks(); + objects = new Map(); + await postgres.query("DELETE FROM chat_attachments"); + await postgres.query("DELETE FROM chat_attachment_usage"); + await postgres.query("DELETE FROM chat_message_deletions"); + await postgres.query( + `DELETE FROM settings WHERE name = 'chat_attachment_daily_mb'`, + ); + await postgres.query("DELETE FROM direct_messages"); + await postgres.query("DELETE FROM direct_conversations"); + await postgres.query("DELETE FROM players"); + }); + + // An image as the composer leaves it: uploaded, not yet sent. + const uploaded = async ( + steamId: string, + type: ChatLobbyType, + roomId: string, + ): Promise => { + const created = await attachments.create(steamId, type, roomId, { + name: "smoke.png", + size: PNG.length, + mime_type: "image/png", + width: 640, + height: 360, + }); + + if ("code" in created) { + throw new Error(`refused: ${created.code}`); + } + + expect( + await attachments.uploadPart(steamId, created.id, 1, PNG), + ).toBeNull(); + + const completed = await attachments.complete(steamId, created.id); + if ("code" in completed) { + throw new Error(`refused: ${completed.code}`); + } + + return created.id; + }; + + const row = async (id: string) => + ( + await postgres.query< + Array<{ + message_id: string | null; + expires_at: Date | null; + storage_prefix: string; + room_type: string; + room_id: string; + }> + >( + `SELECT message_id::text AS message_id, expires_at, storage_prefix, + room_type, room_id + FROM chat_attachments WHERE id = $1::uuid`, + [id], + ) + ).at(0); + + const sendDirect = (roomId: string, from: string, ids: string[], text = "") => + chat.sendMessageToChat( + ChatLobbyType.Direct, + roomId, + { steam_id: from, name: "Someone", role: "user" } as any, + text, + true, + "web", + { attachments: ids }, + ); + + describe("uploading", () => { + it("files an upload under its room's scope and day, and expires it in a day", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + + const stored = await row(id); + + expect(stored.storage_prefix).toMatch( + new RegExp(`^chat-attachments/rooms/\\d{4}-\\d{2}-\\d{2}/${id}/$`), + ); + expect(stored.message_id).toBeNull(); + expect(stored.expires_at.getTime()).toBeGreaterThan( + Date.now() + 23 * 60 * 60 * 1000, + ); + expect(objects.has(`${stored.storage_prefix}file`)).toBe(true); + }); + + it("refuses a first part that is not what it claimed to be", async () => { + const me = await fx.player(); + const created = await attachments.create( + me, + ChatLobbyType.MatchMaking, + "lobby-1", + { name: "smoke.png", size: 64, mime_type: "image/png" }, + ); + + if ("code" in created) { + throw new Error("refused"); + } + + await expect( + attachments.uploadPart( + me, + created.id, + 1, + Buffer.concat([Buffer.from(""), Buffer.alloc(58)]), + ), + ).resolves.toBe(ChatErrorCode.UnsupportedType); + + expect(s3.uploadPart).not.toHaveBeenCalled(); + }); + + it("refuses a part of the wrong length", async () => { + const me = await fx.player(); + const created = await attachments.create( + me, + ChatLobbyType.MatchMaking, + "lobby-1", + { name: "smoke.png", size: 64, mime_type: "image/png" }, + ); + + if ("code" in created) { + throw new Error("refused"); + } + + await expect( + attachments.uploadPart(me, created.id, 1, PNG.subarray(0, 10)), + ).resolves.toBe(ChatErrorCode.Invalid); + }); + + it("lets nobody but the uploader add to an upload", async () => { + const me = await fx.player(); + const other = await fx.player(); + const created = await attachments.create( + me, + ChatLobbyType.MatchMaking, + "lobby-1", + { name: "smoke.png", size: PNG.length, mime_type: "image/png" }, + ); + + if ("code" in created) { + throw new Error("refused"); + } + + await expect( + attachments.uploadPart(other, created.id, 1, PNG), + ).resolves.toBe(ChatErrorCode.NotFound); + }); + + it("caps how many unsent uploads one player can hold", async () => { + const me = await fx.player(); + + for (let i = 0; i < ChatAttachmentsService.MAX_PENDING_PER_PLAYER; i++) { + await attachments.create(me, ChatLobbyType.MatchMaking, "lobby-1", { + name: "smoke.png", + size: PNG.length, + mime_type: "image/png", + }); + } + + await expect( + attachments.create(me, ChatLobbyType.MatchMaking, "lobby-1", { + name: "smoke.png", + size: PNG.length, + mime_type: "image/png", + }), + ).resolves.toEqual({ code: ChatErrorCode.TooManyPending }); + }); + + const start = (steamId: string, size: number) => + attachments.create(steamId, ChatLobbyType.MatchMaking, "lobby-1", { + name: "clip.mp4", + size, + mime_type: "video/mp4", + }); + + const setDailyMb = (megabytes: number) => + postgres.query( + `INSERT INTO settings (name, value) VALUES ('chat_attachment_daily_mb', $1) + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value`, + [String(megabytes)], + ); + + it("holds a player to the operator's daily allowance", async () => { + const me = await fx.player(); + await setDailyMb(1); + + expect(await start(me, 600 * 1024)).not.toHaveProperty("code"); + await expect(start(me, 600 * 1024)).resolves.toEqual({ + code: ChatErrorCode.QuotaExceeded, + }); + }); + + // Otherwise uploading and removing the same file over and over is free. + it("counts an upload against the allowance even once it is gone", async () => { + const me = await fx.player(); + await setDailyMb(1); + + const first = await start(me, 600 * 1024); + if ("code" in first) { + throw new Error("refused"); + } + await attachments.discard(me, first.id); + + await expect(start(me, 600 * 1024)).resolves.toEqual({ + code: ChatErrorCode.QuotaExceeded, + }); + }); + + it("only counts the last day against the allowance", async () => { + const me = await fx.player(); + await setDailyMb(1); + + await start(me, 600 * 1024); + await postgres.query( + `UPDATE chat_attachment_usage SET created_at = now() - interval '25 hours'`, + ); + + expect(await start(me, 600 * 1024)).not.toHaveProperty("code"); + }); + + it("never lets a burst of uploads past the caps", async () => { + const me = await fx.player(); + const attempts = ChatAttachmentsService.MAX_PENDING_PER_PLAYER + 5; + + const results = await Promise.all( + Array.from({ length: attempts }, () => start(me, 1024)), + ); + + expect(results.filter((result) => !("code" in result))).toHaveLength( + ChatAttachmentsService.MAX_PENDING_PER_PLAYER, + ); + expect( + results.filter( + (result) => + "code" in result && result.code === ChatErrorCode.TooManyPending, + ), + ).toHaveLength(5); + }); + + it("refuses an image over the pixel cap, whatever it claimed", async () => { + const me = await fx.player(); + const huge = Buffer.concat([ + PNG.subarray(0, 16), + uint32(10_000), + uint32(5_000), + Buffer.alloc(40), + ]); + const created = await attachments.create( + me, + ChatLobbyType.MatchMaking, + "lobby-1", + { name: "huge.png", size: huge.length, mime_type: "image/png" }, + ); + + if ("code" in created) { + throw new Error("refused"); + } + + await expect( + attachments.uploadPart( + me, + created.id, + 1, + Readable.from([huge]), + huge.length, + ), + ).resolves.toBe(ChatErrorCode.TooLarge); + expect(s3.uploadPart).not.toHaveBeenCalled(); + }); + + it("removes an upload taken out of the tray straight away", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + const { storage_prefix } = await row(id); + + await expect(attachments.discard(me, id)).resolves.toBe(true); + + expect(await row(id)).toBeUndefined(); + expect(s3.removePrefixStrictly).toHaveBeenCalledWith(storage_prefix); + }); + + it("never lets a sent file be pulled out from under its message", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + + await attachments.claim([id], { + type: ChatLobbyType.MatchMaking, + roomId: "lobby-1", + steamId: me, + messageId: randomUUID(), + expiresAt: new Date(Date.now() + 3600_000), + }); + + await expect(attachments.discard(me, id)).resolves.toBe(false); + expect(await row(id)).toBeDefined(); + }); + }); + + describe("claiming", () => { + const claimAs = (steamId: string, ids: string[], roomId = "lobby-1") => + attachments.claim(ids, { + type: ChatLobbyType.MatchMaking, + roomId, + steamId, + messageId: randomUUID(), + expiresAt: new Date(Date.now() + 3600_000), + }); + + it("hands back the files in the order they were attached", async () => { + const me = await fx.player(); + const first = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + const second = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + + const claimed = await claimAs(me, [second, first]); + + expect(claimed.map(({ id }) => id)).toEqual([second, first]); + expect(claimed[0]).toEqual({ + id: second, + kind: "image", + name: "smoke.png", + mime_type: "image/png", + size: PNG.length, + width: 640, + height: 360, + }); + }); + + it("claims nothing when one file is someone else's", async () => { + const me = await fx.player(); + const other = await fx.player(); + const mine = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + const theirs = await uploaded( + other, + ChatLobbyType.MatchMaking, + "lobby-1", + ); + + await expect(claimAs(me, [mine, theirs])).resolves.toBeNull(); + expect((await row(mine)).message_id).toBeNull(); + }); + + it("claims nothing uploaded for another room", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-2"); + + await expect(claimAs(me, [id], "lobby-1")).resolves.toBeNull(); + }); + + it("claims nothing still uploading", async () => { + const me = await fx.player(); + const created = await attachments.create( + me, + ChatLobbyType.MatchMaking, + "lobby-1", + { name: "smoke.png", size: PNG.length, mime_type: "image/png" }, + ); + + if ("code" in created) { + throw new Error("refused"); + } + + await expect(claimAs(me, [created.id])).resolves.toBeNull(); + }); + + it("claims a file only once", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + + expect(await claimAs(me, [id])).not.toBeNull(); + await expect(claimAs(me, [id])).resolves.toBeNull(); + }); + + it("knows a file is already sent by its own sender, in this room", async () => { + const me = await fx.player(); + const other = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + const claim = { + type: ChatLobbyType.MatchMaking, + roomId: "lobby-1", + steamId: me, + messageId: randomUUID(), + expiresAt: new Date(Date.now() + 3600_000), + }; + + await expect(attachments.sentBy([id], claim)).resolves.toBe(false); + + await attachments.claim([id], claim); + + await expect(attachments.sentBy([id], claim)).resolves.toBe(true); + await expect( + attachments.sentBy([id], { ...claim, steamId: other }), + ).resolves.toBe(false); + await expect( + attachments.sentBy([id], { ...claim, roomId: "lobby-2" }), + ).resolves.toBe(false); + }); + + it("claims nothing that has expired", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + await postgres.query( + `UPDATE chat_attachments SET expires_at = now() - interval '1 minute'`, + ); + + await expect(claimAs(me, [id])).resolves.toBeNull(); + }); + }); + + describe("direct messages", () => { + it("keeps a direct message's files with it in history", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + const id = await uploaded(me, ChatLobbyType.Direct, room); + + const sent = await sendDirect(room, me, [id]); + expect(sent.accepted).toBe(true); + + const [message] = await chat["getMessages"](ChatLobbyType.Direct, room); + + expect(message.message).toBe(""); + expect(message.attachments).toEqual([ + expect.objectContaining({ id, kind: "image", mime_type: "image/png" }), + ]); + // Lives as long as the message does, however long that is. + expect((await row(id)).expires_at).toBeNull(); + }); + + it("keeps a direct message's GIF with it in history", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + await postgres.query( + `INSERT INTO settings (name, value) VALUES ('giphy_api_key', 'k') + ON CONFLICT (name) DO UPDATE SET value = EXCLUDED.value`, + ); + + await chat.sendMessageToChat( + ChatLobbyType.Direct, + room, + { steam_id: me, name: "Someone", role: "user" } as any, + "", + true, + "web", + { gif: { id: "abc123", width: 480, height: 270 } }, + ); + + await postgres.query(`DELETE FROM settings WHERE name = 'giphy_api_key'`); + + const [message] = await chat["getMessages"](ChatLobbyType.Direct, room); + + expect(message.gif).toEqual({ id: "abc123", width: 480, height: 270 }); + }); + + it("takes the claim back when the message is not stored", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + const id = await uploaded(me, ChatLobbyType.Direct, room); + await postgres.query( + `INSERT INTO player_blocks (blocker_steam_id, blocked_steam_id) + VALUES ($1::bigint, $2::bigint)`, + [friend, me], + ); + + await expect(sendDirect(room, me, [id])).resolves.toEqual({ + accepted: false, + code: ChatErrorCode.NotAllowed, + }); + + expect((await row(id)).message_id).toBeNull(); + }); + + it("deletes a direct message's files when retention sweeps the message", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + const old = await uploaded(me, ChatLobbyType.Direct, room); + const recent = await uploaded(me, ChatLobbyType.Direct, room); + + await sendDirect(room, me, [old]); + await postgres.query( + `UPDATE direct_messages SET created_at = now() - interval '400 days'`, + ); + await sendDirect(room, me, [recent]); + + const { storage_prefix } = await row(old); + + await new PruneDirectMessages( + logger as any, + postgres, + attachments, + ).process({} as any); + + expect(await row(old)).toBeUndefined(); + expect(s3.removePrefixStrictly).toHaveBeenCalledWith(storage_prefix); + expect(await row(recent)).toBeDefined(); + }); + + it("deletes a direct message's files when its author deletes it", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + const id = await uploaded(me, ChatLobbyType.Direct, room); + const sent = await sendDirect(room, me, [id]); + const { storage_prefix } = await row(id); + + await expect( + chat.deleteMessage( + ChatLobbyType.Direct, + room, + sent.accepted ? sent.messageId : "", + { steam_id: me, name: "Someone", role: "user" } as any, + ), + ).resolves.toEqual({ deleted: true }); + + expect(await row(id)).toBeUndefined(); + expect(s3.removePrefixStrictly).toHaveBeenCalledWith(storage_prefix); + }); + + it("marks the files of a deleted player's messages for removal", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + const id = await uploaded(me, ChatLobbyType.Direct, room); + await sendDirect(room, me, [id]); + + await postgres.query(`DELETE FROM players WHERE steam_id = $1::bigint`, [ + me, + ]); + + const stored = await row(id); + expect(stored.expires_at.getTime()).toBeLessThanOrEqual(Date.now()); + }); + }); + + describe("expiry", () => { + it("removes uploads never sent once their day is up", async () => { + const me = await fx.player(); + const unsent = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + const fresh = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + await postgres.query( + `UPDATE chat_attachments SET expires_at = now() - interval '1 second' + WHERE id = $1::uuid`, + [unsent], + ); + + await expect(attachments.removeExpired()).resolves.toBe(1); + + expect(await row(unsent)).toBeUndefined(); + expect(await row(fresh)).toBeDefined(); + }); + + it("never times out a sent direct message's file on its own", async () => { + const me = await fx.player(); + const friend = await fx.player(); + const room = directRoomId(me, friend); + const id = await uploaded(me, ChatLobbyType.Direct, room); + await sendDirect(room, me, [id]); + + await attachments.removeExpired(); + + expect(await row(id)).toBeDefined(); + }); + + const sendToLobby = async (steamId: string, ids: string[]) => { + const sent = await chat.sendMessageToChat( + ChatLobbyType.MatchMaking, + "lobby-1", + { steam_id: steamId, name: "Someone", role: "user" } as any, + "", + true, + "web", + { attachments: ids }, + ); + + if (sent.accepted === false) { + throw new Error(`refused: ${sent.code}`); + } + + return sent.messageId; + }; + + it("keeps a deleted lobby message's files as evidence until they expire", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.MatchMaking, "lobby-1"); + const messageId = await sendToLobby(me, [id]); + const [stored] = redis.hset.mock.calls.at(-1).slice(2); + + redis.hget.mockImplementation(async (key: string, field: string) => + key === "chat_matchmaking_lobby-1" && field === messageId + ? stored + : null, + ); + + await expect( + chat.deleteMessage(ChatLobbyType.MatchMaking, "lobby-1", messageId, { + steam_id: me, + name: "Someone", + role: "moderator", + } as any), + ).resolves.toEqual({ deleted: true }); + + redis.hget.mockResolvedValue(null); + + const [audit] = await postgres.query< + Array<{ attachments: Array<{ id: string }> | null }> + >(`SELECT attachments FROM chat_message_deletions`); + expect(audit.attachments.map(({ id: kept }) => kept)).toEqual([id]); + + const kept = await attachments.find(id); + expect(kept?.deleted_at).not.toBeNull(); + expect(s3.removePrefixStrictly).not.toHaveBeenCalled(); + + await attachments.removeExpired(); + expect(await attachments.find(id)).toBeDefined(); + + await postgres.query( + `UPDATE chat_attachments SET expires_at = now() - interval '1 second'`, + ); + await attachments.removeExpired(); + + expect(await row(id)).toBeUndefined(); + }); + + it("never brings a deleted or due file back by moving it to the match", async () => { + const me = await fx.player(); + const deleted = await uploaded(me, ChatLobbyType.Draft, "draft-1"); + const due = await uploaded(me, ChatLobbyType.Draft, "draft-1"); + + for (const id of [deleted, due]) { + await attachments.claim([id], { + type: ChatLobbyType.Draft, + roomId: "draft-1", + steamId: me, + messageId: randomUUID(), + expiresAt: new Date(Date.now() + 60_000), + }); + } + + await postgres.query( + `UPDATE chat_attachments SET deleted_at = now() WHERE id = $1::uuid`, + [deleted], + ); + await postgres.query( + `UPDATE chat_attachments SET expires_at = now() - interval '1 second' + WHERE id = $1::uuid`, + [due], + ); + + await attachments.moveRoom( + ChatLobbyType.Draft, + "draft-1", + ChatLobbyType.Match, + "m-1", + new Date(Date.now() + 3600_000), + ); + + for (const id of [deleted, due]) { + const stored = await row(id); + expect(stored.room_type).toBe("draft"); + expect(stored.expires_at.getTime()).toBeLessThan(Date.now() + 61_000); + } + }); + + it("moves a draft's files into its match, and keeps them as long as the match chat", async () => { + const me = await fx.player(); + const id = await uploaded(me, ChatLobbyType.Draft, "draft-1"); + await attachments.claim([id], { + type: ChatLobbyType.Draft, + roomId: "draft-1", + steamId: me, + messageId: randomUUID(), + expiresAt: new Date(Date.now() + 60_000), + }); + + const later = new Date(Date.now() + 3600_000); + await attachments.moveRoom( + ChatLobbyType.Draft, + "draft-1", + ChatLobbyType.Match, + "m-1", + later, + ); + + const moved = await row(id); + expect(moved).toMatchObject({ room_type: "match", room_id: "m-1" }); + expect(moved.expires_at.getTime()).toBe(later.getTime()); + }); + }); +}); diff --git a/test/chat-blocks.spec.ts b/test/chat-blocks.spec.ts index 6cac31d5..6bc05310 100644 --- a/test/chat-blocks.spec.ts +++ b/test/chat-blocks.spec.ts @@ -123,6 +123,13 @@ describe("chat blocks (SQL-driven)", () => { editChatMessage: async () => {}, } as any, blocks, + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }, 600_000); diff --git a/test/chat-direct-messages.spec.ts b/test/chat-direct-messages.spec.ts index 318a9a21..c56fedbd 100644 --- a/test/chat-direct-messages.spec.ts +++ b/test/chat-direct-messages.spec.ts @@ -64,6 +64,13 @@ describe("direct messages (SQL-driven)", () => { { getConnection: () => redis } as any, push as any, new PlayerBlocksService(postgres), + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }, 600_000); @@ -335,7 +342,10 @@ describe("direct messages (SQL-driven)", () => { }); describe("retention", () => { - const prune = () => new PruneDirectMessages(logger as any, postgres); + const prune = () => + new PruneDirectMessages(logger as any, postgres, { + removeExpired: jest.fn(async () => 0), + } as any); const setRetention = (days: number) => postgres.query( @@ -783,7 +793,9 @@ describe("direct messages (SQL-driven)", () => { `UPDATE direct_messages SET created_at = now() - interval '400 days'`, ); - await new PruneDirectMessages(logger as any, postgres).process({} as any); + await new PruneDirectMessages(logger as any, postgres, { + removeExpired: jest.fn(async () => 0), + } as any).process({} as any); expect(await rows()).toEqual([]); }); diff --git a/test/chat-moderation.spec.ts b/test/chat-moderation.spec.ts index 58157ece..19d42c00 100644 --- a/test/chat-moderation.spec.ts +++ b/test/chat-moderation.spec.ts @@ -112,6 +112,13 @@ describe("chat moderation (SQL-driven)", () => { { getConnection: () => redis } as any, push, new PlayerBlocksService(postgres), + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }, 600_000); diff --git a/test/chat-redis-actions.spec.ts b/test/chat-redis-actions.spec.ts index d613e954..d0cc90f8 100644 --- a/test/chat-redis-actions.spec.ts +++ b/test/chat-redis-actions.spec.ts @@ -83,6 +83,13 @@ describe("chat edits and self deletes (SQL-driven)", () => { { getConnection: () => redis } as any, push as any, new PlayerBlocksService(postgres), + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }, 600_000); @@ -1134,6 +1141,13 @@ describe("chat edits and self deletes (SQL-driven)", () => { { getConnection: () => redis } as any, push as any, new PlayerBlocksService(postgres), + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); return { gateway: new ChatGateway(service), hasura: stub, rcon, connect }; diff --git a/test/match-chat-archive.spec.ts b/test/match-chat-archive.spec.ts index e3a80fa9..2cbc3146 100644 --- a/test/match-chat-archive.spec.ts +++ b/test/match-chat-archive.spec.ts @@ -121,6 +121,14 @@ describe("match chat archive (SQL-driven)", () => { editChatMessage: jest.fn(async () => {}), } as any, new PlayerBlocksService(postgres), + { + claim: jest.fn(), + sentBy: jest.fn(async () => false), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }, 600_000); diff --git a/test/tournament-chat-window.spec.ts b/test/tournament-chat-window.spec.ts index 31b51963..e5af85e6 100644 --- a/test/tournament-chat-window.spec.ts +++ b/test/tournament-chat-window.spec.ts @@ -67,6 +67,13 @@ describe("finished tournament chat (SQL-driven)", () => { { getConnection: () => redis } as any, { sendChatMessage: jest.fn(async () => {}) } as any, new PlayerBlocksService(postgres), + { + claim: jest.fn(), + expireMessage: jest.fn(async () => {}), + markDeleted: jest.fn(async () => {}), + moveRoom: jest.fn(async () => {}), + } as any, + { enabled: jest.fn(async () => false) } as any, ); }, 600_000);