diff --git a/backend/rest-api/src/routes/server/get_server.rs b/backend/rest-api/src/routes/server/get_server.rs index 82b7ba1..83eccc9 100644 --- a/backend/rest-api/src/routes/server/get_server.rs +++ b/backend/rest-api/src/routes/server/get_server.rs @@ -21,7 +21,7 @@ pub(super) async fn get_mine_server( .repository .get_servers_of_user_without_favicon(account.id().clone()) .await?; - let mut servers = stream::iter( + let mut servers: Vec = stream::iter( result .into_iter() .filter(|s| account.is_admin() || !s.hidden), diff --git a/backend/rest-api/src/routes/server/router.rs b/backend/rest-api/src/routes/server/router.rs index f5272c7..13184e8 100644 --- a/backend/rest-api/src/routes/server/router.rs +++ b/backend/rest-api/src/routes/server/router.rs @@ -13,9 +13,9 @@ use crate::state::AppState; use crate::utils::rate_limit::ClientIpKeyExtractor; use axum::Router; use axum::routing::{delete, get, patch, post}; -use repository::models::record::RecordData; +use base64::prelude::*; use repository::models::server::Server; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; use std::sync::Arc; use std::time::Duration; use tower_governor::GovernorLayer; @@ -98,18 +98,23 @@ pub fn router() -> Router { pub(super) async fn include_stats( include_stats: bool, state: &AppState, - servers: &mut Vec, + servers: &mut [BiggerServerResponse], ) -> Result<(), AppError> { - if include_stats { - let server_ids: Vec = servers.iter().map(|s| s.server.id).collect(); - - let mut records_map = state - .repository - .get_last_pings_for_servers(&server_ids) - .await?; - for s in servers { - s.data = records_map.remove(&s.server.id); - } + if !include_stats { + return Ok(()); + } + + let server_ids: Vec = servers.iter().map(|s| s.server.id).collect(); + + let mut records_map = state + .repository + .get_last_pings_for_servers(&server_ids) + .await?; + + for server in servers { + server.stats_data = records_map + .remove(&server.server.id) + .map(|e| e.into_binary()); } Ok(()) @@ -121,13 +126,26 @@ pub(super) struct ServerListQueryParams { pub include_owners: Option, } +fn serialize_opt_base64(data: &Option>, serializer: S) -> Result +where + S: Serializer, +{ + match data { + Some(bytes) => serializer.serialize_str(&BASE64_STANDARD.encode(bytes)), + None => serializer.serialize_none(), + } +} + #[derive(Serialize)] pub(super) struct BiggerServerResponse { #[serde(flatten)] pub server: Server, - #[serde(skip_serializing_if = "Option::is_none")] - pub data: Option, + #[serde( + skip_serializing_if = "Option::is_none", + serialize_with = "serialize_opt_base64" + )] + pub stats_data: Option>, #[serde(skip_serializing_if = "Option::is_none")] pub user: Option>, @@ -144,7 +162,7 @@ impl BiggerServerResponse { Self { client_infos: get_client_infos(state, &server).await, server, - data: None, + stats_data: None, user, } } @@ -153,7 +171,7 @@ impl BiggerServerResponse { Self { client_infos: get_client_infos(state, &server).await, server, - data: None, + stats_data: None, user: None, } } diff --git a/backend/rest-api/src/services/clients/laby.rs b/backend/rest-api/src/services/clients/laby.rs index 4f8dcb3..62c6035 100644 --- a/backend/rest-api/src/services/clients/laby.rs +++ b/backend/rest-api/src/services/clients/laby.rs @@ -51,13 +51,11 @@ async fn construct_laby_info(state: &AppState, server: Arc) -> La }; let yt_trailer = match manifest_url(&server) { - Some(url) if is_allowed_manifest_url(url) => { - state - .laby_manifest_cache - .get(&url.to_string()) - .await - .flatten() - } + Some(url) if is_allowed_manifest_url(url) => state + .laby_manifest_cache + .get(&url.to_string()) + .await + .flatten(), _ => None, }; @@ -238,8 +236,7 @@ pub struct Location { } const MANIFEST_CACHE_TTL: Duration = Duration::from_hours(24); -const ALLOWED_MANIFEST_ORIGINS: [&str; 2] = - ["https://dl.labymod.net", "https://resource.laby.net"]; +const ALLOWED_MANIFEST_ORIGINS: [&str; 2] = ["https://dl.labymod.net", "https://resource.laby.net"]; fn manifest_url(server: &RawLabyServer) -> Option<&str> { server @@ -359,10 +356,7 @@ mod tests { fn manifest_url_finds_manifest_attachment() { let server = server_with_attachments(vec![ attachment("background.webp", "https://dl.labymod.net/bg.webp"), - attachment( - "manifest.json", - "https://dl.labymod.net/manifest.json", - ), + attachment("manifest.json", "https://dl.labymod.net/manifest.json"), ]); assert_eq!( manifest_url(&server), @@ -387,7 +381,9 @@ mod tests { assert!(is_allowed_manifest_url( "https://resource.laby.net/abc/manifest.json" )); - assert!(!is_allowed_manifest_url("http://dl.labymod.net/manifest.json")); + assert!(!is_allowed_manifest_url( + "http://dl.labymod.net/manifest.json" + )); assert!(!is_allowed_manifest_url( "https://dl.labymod.net.evil.com/manifest.json" )); @@ -404,7 +400,10 @@ mod tests { fn parses_manifest_trailer() { let manifest: RawLabyManifest = serde_json::from_str(r#"{"yt_trailer": "dQw4w9WgXcQ"}"#).unwrap(); - assert_eq!(normalize_trailer(manifest.yt_trailer), Some("dQw4w9WgXcQ".to_string())); + assert_eq!( + normalize_trailer(manifest.yt_trailer), + Some("dQw4w9WgXcQ".to_string()) + ); let manifest: RawLabyManifest = serde_json::from_str(r#"{}"#).unwrap(); assert_eq!(normalize_trailer(manifest.yt_trailer), None); diff --git a/backend/worker/src/tasks/pinger.rs b/backend/worker/src/tasks/pinger.rs index 2940937..1c9e66a 100644 --- a/backend/worker/src/tasks/pinger.rs +++ b/backend/worker/src/tasks/pinger.rs @@ -156,10 +156,8 @@ pub async fn ping_worker(repository: PostgresRepository, state_updater: Sender { - players_online = ping - .players - .as_ref() - .map_or(0, |t| t.online); + players_online = + ping.players.as_ref().map_or(0, |t| t.online); update_server_from_ping( &mut server, diff --git a/frontend/app/core/lib/api/index.ts b/frontend/app/core/lib/api/index.ts index 62adb4c..a80296d 100644 --- a/frontend/app/core/lib/api/index.ts +++ b/frontend/app/core/lib/api/index.ts @@ -1,5 +1,6 @@ export * from "./schemas"; export * from "./client"; +export * from "./records-binary"; export * from "./servers"; export * from "./admin"; export * from "./alerts"; diff --git a/frontend/app/core/lib/api/records-binary.test.ts b/frontend/app/core/lib/api/records-binary.test.ts new file mode 100644 index 0000000..9da65b5 --- /dev/null +++ b/frontend/app/core/lib/api/records-binary.test.ts @@ -0,0 +1,131 @@ +import { describe, it, expect } from "vitest" +import { + base64ToArrayBuffer, + decodeStatsBase64, + parseRecordBinary, +} from "./records-binary" +import { normalizeServerData } from "./servers" + +/** + * Test encoder mirroring the backend (`RecordData::into_binary`): + * u32 LE count, i64 LE base, N x u32 LE deltas, N x i32 LE values. + */ +function encodeRecordBinary(dates: number[], values: number[]): ArrayBuffer { + const len = dates.length + const buffer = new ArrayBuffer(12 + len * 4 + len * 4) + const view = new DataView(buffer) + view.setUint32(0, len, true) + if (len === 0) { + view.setBigInt64(4, 0n, true) + return buffer + } + const base = BigInt(dates[0]) + view.setBigInt64(4, base, true) + for (let i = 0; i < len; i++) { + view.setUint32(12 + i * 4, dates[i] - dates[0], true) + } + for (let i = 0; i < len; i++) { + view.setInt32(12 + len * 4 + i * 4, values[i], true) + } + return buffer +} + +function toBase64(buffer: ArrayBuffer): string { + return Buffer.from(buffer).toString("base64") +} + +describe("parseRecordBinary", () => { + it("returns [] for an empty or truncated buffer", () => { + expect(parseRecordBinary(new ArrayBuffer(0))).toEqual([]) + expect(parseRecordBinary(new ArrayBuffer(3))).toEqual([]) + // Declares 2 points but carries none. + const view = new DataView(new ArrayBuffer(12)) + view.setUint32(0, 2, true) + view.setBigInt64(4, 1_700_000_000n, true) + expect(parseRecordBinary(view.buffer)).toEqual([]) + }) + + it("returns [] when count is zero", () => { + expect(parseRecordBinary(encodeRecordBinary([], []))).toEqual([]) + }) + + it("round-trips timestamps and values", () => { + const dates = [1_700_000_000, 1_700_000_300, 1_700_000_600] + const values = [12, 0, 42] + expect(parseRecordBinary(encodeRecordBinary(dates, values))).toEqual([ + { date: 1_700_000_000, value: 12 }, + { date: 1_700_000_300, value: 0 }, + { date: 1_700_000_600, value: 42 }, + ]) + }) + + it("supports signed values (i32 layout)", () => { + const dates = [1_700_000_000, 1_700_000_060] + expect(parseRecordBinary(encodeRecordBinary(dates, [-1, 7]))).toEqual([ + { date: 1_700_000_000, value: -1 }, + { date: 1_700_000_060, value: 7 }, + ]) + }) +}) + +describe("decodeStatsBase64", () => { + it("returns [] for missing, empty or invalid input", () => { + expect(decodeStatsBase64(null)).toEqual([]) + expect(decodeStatsBase64(undefined)).toEqual([]) + expect(decodeStatsBase64("")).toEqual([]) + expect(decodeStatsBase64("!!!not-base64!!!")).toEqual([]) + }) + + it("decodes the backend base64 payload", () => { + const dates = [1_700_000_000, 1_700_000_300] + const values = [5, 9] + const base64 = toBase64(encodeRecordBinary(dates, values)) + expect(decodeStatsBase64(base64)).toEqual([ + { date: 1_700_000_000, value: 5 }, + { date: 1_700_000_300, value: 9 }, + ]) + }) + + it("base64ToArrayBuffer preserves bytes", () => { + const bytes = new Uint8Array([0, 1, 2, 250, 255]) + const base64 = Buffer.from(bytes).toString("base64") + expect(new Uint8Array(base64ToArrayBuffer(base64))).toEqual(bytes) + }) +}) + +describe("normalizeServerData", () => { + const baseServer = { + id: 1, + name: "test", + ip: "example.com", + port: 25565, + user_id: "user_1", + } + + it("converts stats_data base64 into data records and strips the payload", () => { + const base64 = toBase64(encodeRecordBinary([1_700_000_000, 1_700_000_300], [5, 9])) + const normalized = normalizeServerData({ ...baseServer, stats_data: base64 }) + expect(normalized.data).toEqual([ + { date: 1_700_000_000, value: 5 }, + { date: 1_700_000_300, value: 9 }, + ]) + expect("stats_data" in normalized).toBe(false) + }) + + it("keeps the legacy [dates, values] tuple working", () => { + const normalized = normalizeServerData({ + ...baseServer, + data: [[100, 200], [1, 2]], + }) + expect(normalized.data).toEqual([ + { date: 100, value: 1 }, + { date: 200, value: 2 }, + ]) + }) + + it("leaves already-normalized records untouched", () => { + const records = [{ date: 100, value: 1 }] + const normalized = normalizeServerData({ ...baseServer, data: records }) + expect(normalized.data).toEqual(records) + }) +}) diff --git a/frontend/app/core/lib/api/records-binary.ts b/frontend/app/core/lib/api/records-binary.ts new file mode 100644 index 0000000..86f4610 --- /dev/null +++ b/frontend/app/core/lib/api/records-binary.ts @@ -0,0 +1,80 @@ +import type { ServerRecord } from "./schemas"; + +/** + * Binary layout produced by the backend (`RecordData::into_binary`, + * `backend/crates/repository/src/models/record.rs`) and served both by + * `GET /records/{id}` (`application/octet-stream`) and — base64 encoded — + * by the server list (`stats_data` field of `BigServerResponse`). + * + * ``` + * u32 LE ............ number of points N + * i64 LE ............ base unix timestamp (0 when N == 0) + * N x u32 LE ........ per-point deltas (timestamp - base) + * N x i32 LE ........ per-point values + * ``` + * + * All multi-byte integers are little-endian. + */ + +const COUNT_SIZE = 4; +const BASE_TIMESTAMP_SIZE = 8; +const HEADER_SIZE = COUNT_SIZE + BASE_TIMESTAMP_SIZE; +const DELTA_SIZE = 4; +const VALUE_SIZE = 4; + +/** Decode a base64 string into an `ArrayBuffer` (Node + browser safe). */ +export function base64ToArrayBuffer(base64: string): ArrayBuffer { + const nodeBuffer = (globalThis as { Buffer?: { from(s: string, e: string): Uint8Array } }).Buffer; + if (nodeBuffer) { + const bytes = nodeBuffer.from(base64, "base64"); + // Slice to detach from a potentially pooled underlying ArrayBuffer. + return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + } + const binary = atob(base64); + const bytes = new Uint8Array(binary.length); + for (let i = 0; i < binary.length; i++) { + bytes[i] = binary.charCodeAt(i); + } + return bytes.buffer; +} + +/** + * Parse the raw stats binary format into `{ date, value }` records. + * Tolerant by design: any truncated or malformed payload yields `[]` + * instead of throwing, so a single corrupt server can't break the list. + */ +export function parseRecordBinary(buffer: ArrayBuffer): ServerRecord[] { + if (buffer.byteLength < COUNT_SIZE) return []; + + const view = new DataView(buffer); + const count = view.getUint32(0, true); + if (count === 0) return []; + + const expectedSize = HEADER_SIZE + count * DELTA_SIZE + count * VALUE_SIZE; + if (buffer.byteLength < expectedSize) return []; + + const baseTimestamp = Number(view.getBigInt64(COUNT_SIZE, true)); + const deltasOffset = HEADER_SIZE; + const valuesOffset = HEADER_SIZE + count * DELTA_SIZE; + + const records: ServerRecord[] = new Array(count); + for (let i = 0; i < count; i++) { + const delta = view.getUint32(deltasOffset + i * DELTA_SIZE, true); + const value = view.getInt32(valuesOffset + i * VALUE_SIZE, true); + records[i] = { date: baseTimestamp + delta, value }; + } + return records; +} + +/** + * Decode a base64 `stats_data` payload (server list) into records. + * Returns `[]` for missing, empty or invalid input — never throws. + */ +export function decodeStatsBase64(base64: string | null | undefined): ServerRecord[] { + if (!base64 || typeof base64 !== "string" || base64.length === 0) return []; + try { + return parseRecordBinary(base64ToArrayBuffer(base64)); + } catch { + return []; + } +} diff --git a/frontend/app/core/lib/api/schemas.ts b/frontend/app/core/lib/api/schemas.ts index 9abfa24..841bfba 100644 --- a/frontend/app/core/lib/api/schemas.ts +++ b/frontend/app/core/lib/api/schemas.ts @@ -36,6 +36,7 @@ export const ServerSchema = z.object({ hidden: z.boolean().nullish().catch(undefined), forced_favicon: z.boolean().nullish().catch(false), registered_date: z.number().nullish().catch(undefined), + stats_data: z.string().nullish().catch(undefined), data: z.array(ServerRecordSchema).nullish().catch(undefined), client_infos: z.any().nullish().catch(undefined), }); diff --git a/frontend/app/core/lib/api/servers.ts b/frontend/app/core/lib/api/servers.ts index 5f3a624..aca0bc3 100644 --- a/frontend/app/core/lib/api/servers.ts +++ b/frontend/app/core/lib/api/servers.ts @@ -2,14 +2,27 @@ import { z } from "zod"; import { PUBLIC_API_URL } from "@/core/lib/config"; import { API_BASE, getHeaders } from "./client"; import { ServerSchema, type Server, type ServerRecord } from "./schemas"; +import { decodeStatsBase64, parseRecordBinary } from "./records-binary"; export function getServerIconUrl(serverId: number | string): string { return `${PUBLIC_API_URL}/servers/${serverId}/icon?v=2` } export function normalizeServerData(server: unknown): Server { - if (server && typeof server === 'object' && 'data' in server) { - const s = server as { data?: unknown }; + if (server && typeof server === 'object') { + const s = server as { data?: unknown; stats_data?: unknown }; + // New backend shape: binary stats base64-encoded in `stats_data`. + // Decoded once here so the rest of the app keeps using `data`. + if (typeof s.stats_data === 'string' && s.stats_data.length > 0) { + const records = decodeStatsBase64(s.stats_data); + const { stats_data: _omitted, ...rest } = s as Record; + void _omitted; + return { + ...(rest as object), + data: records + } as Server; + } + // Legacy shape: `data` as a [dates, values] tuple. if (Array.isArray(s.data) && s.data.length >= 2) { const dataArr = s.data as [number[], number[]]; const dates = dataArr[0] || [] @@ -118,37 +131,18 @@ export async function fetchRecords( if (!res.ok) return [] const buffer = await res.arrayBuffer() - if (buffer.byteLength < 4) return [] - - const dataView = new DataView(buffer) - const len = dataView.getUint32(0, true) - if (len === 0) return [] - - const baseTimestamp = Number(dataView.getBigInt64(4, true)) - - const deltasOffset = 12 - const valuesOffset = 12 + len * 4 - - const deltas = new Uint32Array(buffer, deltasOffset, len) - const valuesArr = new Uint32Array(buffer, valuesOffset, len) - - const dates = new Float64Array(len) - for (let i = 0; i < len; i++) { - dates[i] = baseTimestamp + deltas[i] - } + const records = parseRecordBinary(buffer) if (interval && interval > 0) { const intervalSec = interval / 1000 const buckets: Record = {} - for (let i = 0; i < len; i++) { - const t = dates[i] - const val = valuesArr[i] - const bucketTime = Math.floor(t / intervalSec) * intervalSec + for (const record of records) { + const bucketTime = Math.floor(record.date / intervalSec) * intervalSec if (!buckets[bucketTime]) { buckets[bucketTime] = { sum: 0, count: 0 } } - buckets[bucketTime].sum += val + buckets[bucketTime].sum += record.value buckets[bucketTime].count += 1 } @@ -161,13 +155,6 @@ export async function fetchRecords( } }).sort((a, b) => a.date - b.date) } else { - const records: ServerRecord[] = [] - for (let i = 0; i < len; i++) { - records.push({ - date: dates[i], - value: valuesArr[i] - }) - } return records } } catch (error: unknown) {