Implement binary serialization for prestats and frontend adaptation

backend/rest-api/src/routes/server/get_server.rs

@@ -21,7 +21,7 @@ pub(super) async fn get_mine_server(
         .repository
         .get_servers_of_user_without_favicon(account.id().clone())
         .await?;
-    let mut servers = stream::iter(
+    let mut servers: Vec<BiggerServerResponse> = stream::iter(
         result
             .into_iter()
             .filter(|s| account.is_admin() || !s.hidden),

backend/rest-api/src/routes/server/router.rs

@@ -13,9 +13,9 @@ use crate::state::AppState;
 use crate::utils::rate_limit::ClientIpKeyExtractor;
 use axum::Router;
 use axum::routing::{delete, get, patch, post};
-use repository::models::record::RecordData;
+use base64::prelude::*;
 use repository::models::server::Server;
-use serde::{Deserialize, Serialize};
+use serde::{Deserialize, Serialize, Serializer};
 use std::sync::Arc;
 use std::time::Duration;
 use tower_governor::GovernorLayer;
@@ -98,18 +98,23 @@ pub fn router() -> Router<AppState> {
 pub(super) async fn include_stats(
     include_stats: bool,
     state: &AppState,
-    servers: &mut Vec<BiggerServerResponse>,
+    servers: &mut [BiggerServerResponse],
 ) -> Result<(), AppError> {
-    if include_stats {
-        let server_ids: Vec<u32> = servers.iter().map(|s| s.server.id).collect();
-
-        let mut records_map = state
-            .repository
-            .get_last_pings_for_servers(&server_ids)
-            .await?;
-        for s in servers {
-            s.data = records_map.remove(&s.server.id);
-        }
+    if !include_stats {
+        return Ok(());
+    }
+
+    let server_ids: Vec<u32> = servers.iter().map(|s| s.server.id).collect();
+
+    let mut records_map = state
+        .repository
+        .get_last_pings_for_servers(&server_ids)
+        .await?;
+
+    for server in servers {
+        server.stats_data = records_map
+            .remove(&server.server.id)
+            .map(|e| e.into_binary());
     }
 
     Ok(())
@@ -121,13 +126,26 @@ pub(super) struct ServerListQueryParams {
     pub include_owners: Option<bool>,
 }
 
+fn serialize_opt_base64<S>(data: &Option<Vec<u8>>, serializer: S) -> Result<S::Ok, S::Error>
+where
+    S: Serializer,
+{
+    match data {
+        Some(bytes) => serializer.serialize_str(&BASE64_STANDARD.encode(bytes)),
+        None => serializer.serialize_none(),
+    }
+}
+
 #[derive(Serialize)]
 pub(super) struct BiggerServerResponse {
     #[serde(flatten)]
     pub server: Server,
 
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub data: Option<RecordData>,
+    #[serde(
+        skip_serializing_if = "Option::is_none",
+        serialize_with = "serialize_opt_base64"
+    )]
+    pub stats_data: Option<Vec<u8>>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub user: Option<Arc<ClerkUser>>,
@@ -144,7 +162,7 @@ impl BiggerServerResponse {
         Self {
             client_infos: get_client_infos(state, &server).await,
             server,
-            data: None,
+            stats_data: None,
             user,
         }
     }
@@ -153,7 +171,7 @@ impl BiggerServerResponse {
         Self {
             client_infos: get_client_infos(state, &server).await,
             server,
-            data: None,
+            stats_data: None,
             user: None,
         }
     }

backend/rest-api/src/services/clients/laby.rs

@@ -51,13 +51,11 @@ async fn construct_laby_info(state: &AppState, server: Arc<RawLabyServer>) -> La
     };
 
     let yt_trailer = match manifest_url(&server) {
-        Some(url) if is_allowed_manifest_url(url) => {
-            state
-                .laby_manifest_cache
-                .get(&url.to_string())
-                .await
-                .flatten()
-        }
+        Some(url) if is_allowed_manifest_url(url) => state
+            .laby_manifest_cache
+            .get(&url.to_string())
+            .await
+            .flatten(),
         _ => None,
     };
 
@@ -238,8 +236,7 @@ pub struct Location {
 }
 
 const MANIFEST_CACHE_TTL: Duration = Duration::from_hours(24);
-const ALLOWED_MANIFEST_ORIGINS: [&str; 2] =
-    ["https://dl.labymod.net", "https://resource.laby.net"];
+const ALLOWED_MANIFEST_ORIGINS: [&str; 2] = ["https://dl.labymod.net", "https://resource.laby.net"];
 
 fn manifest_url(server: &RawLabyServer) -> Option<&str> {
     server
@@ -359,10 +356,7 @@ mod tests {
     fn manifest_url_finds_manifest_attachment() {
         let server = server_with_attachments(vec![
             attachment("background.webp", "https://dl.labymod.net/bg.webp"),
-            attachment(
-                "manifest.json",
-                "https://dl.labymod.net/manifest.json",
-            ),
+            attachment("manifest.json", "https://dl.labymod.net/manifest.json"),
         ]);
         assert_eq!(
             manifest_url(&server),
@@ -387,7 +381,9 @@ mod tests {
         assert!(is_allowed_manifest_url(
             "https://resource.laby.net/abc/manifest.json"
         ));
-        assert!(!is_allowed_manifest_url("http://dl.labymod.net/manifest.json"));
+        assert!(!is_allowed_manifest_url(
+            "http://dl.labymod.net/manifest.json"
+        ));
         assert!(!is_allowed_manifest_url(
             "https://dl.labymod.net.evil.com/manifest.json"
         ));
@@ -404,7 +400,10 @@ mod tests {
     fn parses_manifest_trailer() {
         let manifest: RawLabyManifest =
             serde_json::from_str(r#"{"yt_trailer": "dQw4w9WgXcQ"}"#).unwrap();
-        assert_eq!(normalize_trailer(manifest.yt_trailer), Some("dQw4w9WgXcQ".to_string()));
+        assert_eq!(
+            normalize_trailer(manifest.yt_trailer),
+            Some("dQw4w9WgXcQ".to_string())
+        );
 
         let manifest: RawLabyManifest = serde_json::from_str(r#"{}"#).unwrap();
         assert_eq!(normalize_trailer(manifest.yt_trailer), None);

backend/worker/src/tasks/pinger.rs

@@ -156,10 +156,8 @@ pub async fn ping_worker(repository: PostgresRepository, state_updater: Sender<W
                                         .await
                                     {
                                         Ok(ping) => {
-                                            players_online = ping
-                                                .players
-                                                .as_ref()
-                                                .map_or(0, |t| t.online);
+                                            players_online =
+                                                ping.players.as_ref().map_or(0, |t| t.online);
 
                                             update_server_from_ping(
                                                 &mut server,

frontend/app/core/lib/api/index.ts

@@ -1,5 +1,6 @@
 export * from "./schemas";
 export * from "./client";
+export * from "./records-binary";
 export * from "./servers";
 export * from "./admin";
 export * from "./alerts";

frontend/app/core/lib/api/records-binary.test.ts

@@ -0,0 +1,131 @@
+import { describe, it, expect } from "vitest"
+import {
+    base64ToArrayBuffer,
+    decodeStatsBase64,
+    parseRecordBinary,
+} from "./records-binary"
+import { normalizeServerData } from "./servers"
+
+/**
+ * Test encoder mirroring the backend (`RecordData::into_binary`):
+ * u32 LE count, i64 LE base, N x u32 LE deltas, N x i32 LE values.
+ */
+function encodeRecordBinary(dates: number[], values: number[]): ArrayBuffer {
+    const len = dates.length
+    const buffer = new ArrayBuffer(12 + len * 4 + len * 4)
+    const view = new DataView(buffer)
+    view.setUint32(0, len, true)
+    if (len === 0) {
+        view.setBigInt64(4, 0n, true)
+        return buffer
+    }
+    const base = BigInt(dates[0])
+    view.setBigInt64(4, base, true)
+    for (let i = 0; i < len; i++) {
+        view.setUint32(12 + i * 4, dates[i] - dates[0], true)
+    }
+    for (let i = 0; i < len; i++) {
+        view.setInt32(12 + len * 4 + i * 4, values[i], true)
+    }
+    return buffer
+}
+
+function toBase64(buffer: ArrayBuffer): string {
+    return Buffer.from(buffer).toString("base64")
+}
+
+describe("parseRecordBinary", () => {
+    it("returns [] for an empty or truncated buffer", () => {
+        expect(parseRecordBinary(new ArrayBuffer(0))).toEqual([])
+        expect(parseRecordBinary(new ArrayBuffer(3))).toEqual([])
+        // Declares 2 points but carries none.
+        const view = new DataView(new ArrayBuffer(12))
+        view.setUint32(0, 2, true)
+        view.setBigInt64(4, 1_700_000_000n, true)
+        expect(parseRecordBinary(view.buffer)).toEqual([])
+    })
+
+    it("returns [] when count is zero", () => {
+        expect(parseRecordBinary(encodeRecordBinary([], []))).toEqual([])
+    })
+
+    it("round-trips timestamps and values", () => {
+        const dates = [1_700_000_000, 1_700_000_300, 1_700_000_600]
+        const values = [12, 0, 42]
+        expect(parseRecordBinary(encodeRecordBinary(dates, values))).toEqual([
+            { date: 1_700_000_000, value: 12 },
+            { date: 1_700_000_300, value: 0 },
+            { date: 1_700_000_600, value: 42 },
+        ])
+    })
+
+    it("supports signed values (i32 layout)", () => {
+        const dates = [1_700_000_000, 1_700_000_060]
+        expect(parseRecordBinary(encodeRecordBinary(dates, [-1, 7]))).toEqual([
+            { date: 1_700_000_000, value: -1 },
+            { date: 1_700_000_060, value: 7 },
+        ])
+    })
+})
+
+describe("decodeStatsBase64", () => {
+    it("returns [] for missing, empty or invalid input", () => {
+        expect(decodeStatsBase64(null)).toEqual([])
+        expect(decodeStatsBase64(undefined)).toEqual([])
+        expect(decodeStatsBase64("")).toEqual([])
+        expect(decodeStatsBase64("!!!not-base64!!!")).toEqual([])
+    })
+
+    it("decodes the backend base64 payload", () => {
+        const dates = [1_700_000_000, 1_700_000_300]
+        const values = [5, 9]
+        const base64 = toBase64(encodeRecordBinary(dates, values))
+        expect(decodeStatsBase64(base64)).toEqual([
+            { date: 1_700_000_000, value: 5 },
+            { date: 1_700_000_300, value: 9 },
+        ])
+    })
+
+    it("base64ToArrayBuffer preserves bytes", () => {
+        const bytes = new Uint8Array([0, 1, 2, 250, 255])
+        const base64 = Buffer.from(bytes).toString("base64")
+        expect(new Uint8Array(base64ToArrayBuffer(base64))).toEqual(bytes)
+    })
+})
+
+describe("normalizeServerData", () => {
+    const baseServer = {
+        id: 1,
+        name: "test",
+        ip: "example.com",
+        port: 25565,
+        user_id: "user_1",
+    }
+
+    it("converts stats_data base64 into data records and strips the payload", () => {
+        const base64 = toBase64(encodeRecordBinary([1_700_000_000, 1_700_000_300], [5, 9]))
+        const normalized = normalizeServerData({ ...baseServer, stats_data: base64 })
+        expect(normalized.data).toEqual([
+            { date: 1_700_000_000, value: 5 },
+            { date: 1_700_000_300, value: 9 },
+        ])
+        expect("stats_data" in normalized).toBe(false)
+    })
+
+    it("keeps the legacy [dates, values] tuple working", () => {
+        const normalized = normalizeServerData({
+            ...baseServer,
+            data: [[100, 200], [1, 2]],
+        })
+        expect(normalized.data).toEqual([
+            { date: 100, value: 1 },
+            { date: 200, value: 2 },
+        ])
+    })
+
+    it("leaves already-normalized records untouched", () => {
+        const records = [{ date: 100, value: 1 }]
+        const normalized = normalizeServerData({ ...baseServer, data: records })
+        expect(normalized.data).toEqual(records)
+    })
+})

frontend/app/core/lib/api/records-binary.ts

@@ -0,0 +1,80 @@
+import type { ServerRecord } from "./schemas";
+
+/**
+ * Binary layout produced by the backend (`RecordData::into_binary`,
+ * `backend/crates/repository/src/models/record.rs`) and served both by
+ * `GET /records/{id}` (`application/octet-stream`) and — base64 encoded —
+ * by the server list (`stats_data` field of `BigServerResponse`).
+ *
+ * ```
+ * u32 LE ............ number of points N
+ * i64 LE ............ base unix timestamp (0 when N == 0)
+ * N x u32 LE ........ per-point deltas (timestamp - base)
+ * N x i32 LE ........ per-point values
+ * ```
+ *
+ * All multi-byte integers are little-endian.
+ */
+
+const COUNT_SIZE = 4;
+const BASE_TIMESTAMP_SIZE = 8;
+const HEADER_SIZE = COUNT_SIZE + BASE_TIMESTAMP_SIZE;
+const DELTA_SIZE = 4;
+const VALUE_SIZE = 4;
+
+/** Decode a base64 string into an `ArrayBuffer` (Node + browser safe). */
+export function base64ToArrayBuffer(base64: string): ArrayBuffer {
+    const nodeBuffer = (globalThis as { Buffer?: { from(s: string, e: string): Uint8Array } }).Buffer;
+    if (nodeBuffer) {
+        const bytes = nodeBuffer.from(base64, "base64");
+        // Slice to detach from a potentially pooled underlying ArrayBuffer.
+        return bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength);
+    }
+    const binary = atob(base64);
+    const bytes = new Uint8Array(binary.length);
+    for (let i = 0; i < binary.length; i++) {
+        bytes[i] = binary.charCodeAt(i);
+    }
+    return bytes.buffer;
+}
+
+/**
+ * Parse the raw stats binary format into `{ date, value }` records.
+ * Tolerant by design: any truncated or malformed payload yields `[]`
+ * instead of throwing, so a single corrupt server can't break the list.
+ */
+export function parseRecordBinary(buffer: ArrayBuffer): ServerRecord[] {
+    if (buffer.byteLength < COUNT_SIZE) return [];
+
+    const view = new DataView(buffer);
+    const count = view.getUint32(0, true);
+    if (count === 0) return [];
+
+    const expectedSize = HEADER_SIZE + count * DELTA_SIZE + count * VALUE_SIZE;
+    if (buffer.byteLength < expectedSize) return [];
+
+    const baseTimestamp = Number(view.getBigInt64(COUNT_SIZE, true));
+    const deltasOffset = HEADER_SIZE;
+    const valuesOffset = HEADER_SIZE + count * DELTA_SIZE;
+
+    const records: ServerRecord[] = new Array(count);
+    for (let i = 0; i < count; i++) {
+        const delta = view.getUint32(deltasOffset + i * DELTA_SIZE, true);
+        const value = view.getInt32(valuesOffset + i * VALUE_SIZE, true);
+        records[i] = { date: baseTimestamp + delta, value };
+    }
+    return records;
+}
+
+/**
+ * Decode a base64 `stats_data` payload (server list) into records.
+ * Returns `[]` for missing, empty or invalid input — never throws.
+ */
+export function decodeStatsBase64(base64: string | null | undefined): ServerRecord[] {
+    if (!base64 || typeof base64 !== "string" || base64.length === 0) return [];
+    try {
+        return parseRecordBinary(base64ToArrayBuffer(base64));
+    } catch {
+        return [];
+    }
+}