From c470acd2d3f763fa62e924b2a33321234bb82b90 Mon Sep 17 00:00:00 2001
From: Anthony Alaribe <anthonyalaribe@gmail.com>
Date: Sun, 25 Oct 2026 23:21:27 +0100
Subject: [PATCH] Stop hourly spike detection loading every pattern's
 error_data

Since a7ece43bc made the hourly per-project jobs actually run, ErrorSpikeDetection
loaded error_data for every open pattern: 4.3 GB of JSON for the self-monitoring
project (HasqlException messages embedding whole SQL statements, up to 21 MB a row),
overflowing the 14G heap and restarting a replica at ~:01 most hours since 10-22.
The query no longer selects it (nothing used it), and stored message/root message/
stack trace are capped so a pattern can't grow that large again.
---
 src/Models/Apis/ErrorPatterns.hs            | 11 ++++++++---
 test/integration/Pages/ErrorPatternsSpec.hs |  3 ++-
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/Models/Apis/ErrorPatterns.hs b/src/Models/Apis/ErrorPatterns.hs
index 82179ff5a..9b0d19fcb 100644
--- a/src/Models/Apis/ErrorPatterns.hs
+++ b/src/Models/Apis/ErrorPatterns.hs
@@ -41,9 +41,11 @@ module Models.Apis.ErrorPatterns (
 )
 where
 
+import Control.Lens ((%~))
 import Data.Aeson qualified as AE
 import Data.Default
 import Data.Effectful.Hasql qualified as Hasql
+import Data.Generics.Product.Fields (field)
 import Data.GeoIP2 qualified as GeoIP2
 import Data.HashMap.Strict qualified as HM
 import Data.IP (IP)
@@ -500,7 +502,6 @@ data ErrorPatternWithCurrentRate = ErrorPatternWithCurrentRate
   , baselineMean :: Maybe Double
   , baselineStddev :: Maybe Double
   , currentHourCount :: Int
-  , errorData :: ATError
   , stacktrace :: Text
   , hash :: Text
   , parentHash :: Maybe Text
@@ -520,7 +521,8 @@ getErrorPatternsWithCurrentRates pid now =
           e.id, e.project_id, e.error_type, LEFT(e.message, 2000), e.service, e.state,
           e.baseline_state, e.baseline_error_rate_mean, e.baseline_error_rate_stddev,
           COALESCE(counts.event_count, 0)::BIGINT AS current_hour_count,
-          e.error_data, LEFT(e.stacktrace, 8000), e.hash, e.parent_hash, e.is_framework, e.slack_thread_ts, e.discord_message_id
+          -- No error_data: one project's was 4.3 GB of JSON, and loading it hourly overflowed the heap.
+          LEFT(e.stacktrace, 8000), e.hash, e.parent_hash, e.is_framework, e.slack_thread_ts, e.discord_message_id
         FROM apis.error_patterns e
         LEFT JOIN apis.error_hourly_stats counts
           ON counts.error_id = e.id AND counts.project_id = e.project_id
@@ -642,7 +644,10 @@ batchUpsertErrorPatterns pid errors now =
     -- is not time-ordered) + sum count (avoids ON CONFLICT duplicate-row error).
     -- The bang keeps the running count forced: a hot hash otherwise accumulates one thunk per occurrence.
     (errs, counts) =
-      V.unzip $ V.fromList $ HM.elems $ HM.fromListWith (\(a, n) (b, !k) -> (bool b a (a.when >= b.when), n + k)) [(e.hash, (e, 1 :: Int)) | e <- V.toList errors]
+      V.unzip $ V.fromList $ HM.elems $ HM.fromListWith (\(a, n) (b, !k) -> (bool b a (a.when >= b.when), n + k)) [(e.hash, (bounded e, 1 :: Int)) | e <- V.toList errors]
+    -- error_data keeps one occurrence for display: a message embedding a whole SQL statement ran to
+    -- 21 MB a row. The hash is already taken, so trimming changes no grouping.
+    bounded e = e & field @"message" %~ T.take 4000 & field @"rootErrorMessage" %~ T.take 4000 & field @"stackTrace" %~ T.take 16000
     -- '' would persist and pass every Maybe check downstream, sending the issue
     -- page off to fetch "trace ''" — every trace-less row in the window.
     traceIds = V.map (mfilter (not . T.null) . (.traceId)) errs
diff --git a/test/integration/Pages/ErrorPatternsSpec.hs b/test/integration/Pages/ErrorPatternsSpec.hs
index 4180b5bc5..8c11face5 100644
--- a/test/integration/Pages/ErrorPatternsSpec.hs
+++ b/test/integration/Pages/ErrorPatternsSpec.hs
@@ -530,7 +530,8 @@ spec = sequential $ aroundAll withTestResources do
           void $ runTestBg regressionTime tr $ ErrorPatterns.updateErrorPatternState errRate.errorId ErrorPatterns.ESResolved regressionTime
           runTestBg regressionTime tr $ BackgroundJobs.runHourlyJob regressionTime 0
           -- Replay the selected error itself; a newly ingested event may have a different fingerprint.
-          runTestBg regressionTime tr $ BackgroundJobs.processProjectErrors pid (V.singleton errRate.errorData{ErrorPatterns.when = regressionTime}) regressionTime
+          pat <- maybe (fail "pattern vanished") pure =<< runTestBg regressionTime tr (ErrorPatterns.getErrorPatternById errRate.errorId)
+          runTestBg regressionTime tr $ BackgroundJobs.processProjectErrors pid (V.singleton pat.errorData{ErrorPatterns.when = regressionTime}) regressionTime
 
           regressedPat <- runTestBg regressionTime tr $ ErrorPatterns.getErrorPatternById errRate.errorId
           fmap (.state) regressedPat `shouldBe` Just ESRegressed