Add RUM audience breakdowns and reusable segments

ci/checks.tsv

@@ -16,6 +16,8 @@
 #
 # Narrow `inputs` only where it is provably sound. A too-wide set costs a rerun;
 # a too-narrow one ships an untested change.
+# Parallel integration, UI, and e2e lanes share dependency installs prepared by
+# the runner before launch; each check still declares its own source inputs.
 frontend	node	@fe @uisrc
 build	ghc	@hs @fe
 doctests	ghc	@hs @fe

e2e/playwright.config.ts

@@ -18,7 +18,7 @@ export default defineConfig({
     trace: "on-first-retry",
   },
   projects: [
-    { name: "chromium", use: { ...devices["Desktop Chrome"] }, testIgnore: /(container-identity|endpoint-analytics|host-map|infrastructure-time-window|metric-exemplars|metrics-catalog|tabs)\.spec\.ts$/ },
-    { name: "chromium-fixtures", use: { ...devices["Desktop Chrome"] }, testMatch: /(container-identity|endpoint-analytics|host-map|infrastructure-time-window|metric-exemplars|metrics-catalog|tabs)\.spec\.ts$/, dependencies: ["chromium"], workers: 1 },
+    { name: "chromium", use: { ...devices["Desktop Chrome"] }, testIgnore: /(container-identity|endpoint-analytics|host-map|infrastructure-time-window|metric-exemplars|metrics-catalog|tabs|rum-audience)\.spec\.ts$/ },
+    { name: "chromium-fixtures", use: { ...devices["Desktop Chrome"] }, testMatch: /(container-identity|endpoint-analytics|host-map|infrastructure-time-window|metric-exemplars|metrics-catalog|tabs|rum-audience)\.spec\.ts$/, dependencies: ["chromium"], workers: 1 },
   ],
 });

e2e/tests/rum-audience.spec.ts

@@ -0,0 +1,62 @@
+import { test, expect } from "@playwright/test";
+import { DEMO_PROJECT, sql } from "./helpers";
+
+test("audience groups filter sessions, search users, and persist named segments", async ({ page }) => {
+  const app = `e2e-rum-audience-${Date.now()}`;
+  const segment = `${app} Mobile Safari`;
+  const cleanup = () => sql(`DELETE FROM otel_logs_and_spans WHERE project_id='${DEMO_PROJECT}' AND resource___service___name='${app}'; DELETE FROM projects.rum_segments WHERE project_id='${DEMO_PROJECT}' AND name='${segment}';`);
+  sql(`INSERT INTO otel_logs_and_spans (project_id,summary,name,timestamp,start_time,end_time,resource___service___name,resource___user_agent___original,attributes___session___id,attributes___user___id,attributes___user___email,attributes___exception___type)
+    VALUES
+      ('${DEMO_PROJECT}',ARRAY['RUM audience'],'documentLoad',now(),now(),now(),'${app}','iPhone Mobile Safari/604','${app}-alice','alice','alice@example.com',NULL),
+      ('${DEMO_PROJECT}',ARRAY['RUM audience'],'browser error',now(),now(),now(),'${app}','iPhone Mobile Safari/604','${app}-alice','alice','alice@example.com','TypeError'),
+      ('${DEMO_PROJECT}',ARRAY['RUM audience'],'documentLoad',now(),now(),now(),'${app}','Windows Chrome/151 Safari/537','${app}-bob','bob','bob@example.com',NULL),
+      ('${DEMO_PROJECT}',ARRAY['RUM audience'],'documentLoad',now(),now(),now(),'${app}',NULL,'${app}-anonymous',NULL,NULL,NULL);`);
+  try {
+    const errors: string[] = [];
+    page.on("pageerror", error => errors.push(error.message));
+    page.on("console", message => { if (message.type() === "error" && /SyntaxError|parse error/i.test(message.text())) errors.push(message.text()); });
+    await page.goto(`/p/${DEMO_PROJECT}/rum?tab=audience&var-app=${app}&since=1H`);
+    const toolbar = page.locator("#rum-segments");
+    await expect(toolbar.getByRole("link", { name: "View sessions", exact: true })).toBeVisible();
+    const browsers = page.getByRole("table").filter({ has: page.getByRole("columnheader", { name: "Browser", exact: true }) });
+    const systems = page.getByRole("table").filter({ has: page.getByRole("columnheader", { name: "Operating system", exact: true }) });
+    await expect(browsers.getByText("Unknown", { exact: true })).toBeVisible();
+    await expect(browsers.locator("tbody tr").filter({ hasText: "Safari" })).toContainText("100 %");
+    for (const width of [1920, 1280, 390]) {
+      await page.setViewportSize({ width, height: 1000 });
+      await expect.poll(async () => {
+        const [browser, system] = await Promise.all([browsers.boundingBox(), systems.boundingBox()]);
+        return !!browser && !!system && (width === 1920 ? Math.abs(browser.y - system.y) < 3 : system.y > browser.y + browser.height);
+      }).toBe(true);
+      for (const table of [browsers, systems]) {
+        expect(await table.evaluate(el => el.getBoundingClientRect().right <= innerWidth && el.scrollWidth <= el.clientWidth + 1)).toBe(true);
+      }
+    }
+    await page.setViewportSize({ width: 1280, height: 1000 });
+    await page.locator("[data-widget]").filter({ has: page.getByText("Identified users", { exact: true }) }).scrollIntoViewIfNeeded();
+    const users = page.getByRole("table").filter({ has: page.getByRole("columnheader", { name: "User", exact: true }) });
+    await expect(users.getByRole("link", { name: "alice@example.com" })).toBeVisible();
+    await page.getByRole("searchbox", { name: "Search Identified users" }).fill("bob");
+    await expect(users.locator("tbody tr").filter({ visible: true })).toHaveCount(1);
+    await expect(users.locator("tbody tr").filter({ visible: true })).toContainText("bob@example.com");
+    await page.getByRole("searchbox", { name: "Search Identified users" }).fill("");
+    await expect(users.locator("tbody tr").filter({ visible: true })).toHaveCount(2);
+    const safari = browsers.locator("tbody tr").filter({ hasText: "Safari" });
+    await safari.focus();
+    await safari.press("Enter");
+    await expect(page).toHaveURL(/var-browser=Safari/);
+    await toolbar.locator("summary").filter({ hasText: "Save segment" }).click();
+    await toolbar.getByLabel("Segment name").fill(segment);
+    await toolbar.getByRole("button", { name: "Save segment", exact: true }).click();
+    await expect(toolbar.getByRole("link", { name: segment, exact: true })).toBeVisible();
+    await page.reload();
+    await expect(toolbar.getByRole("link", { name: segment, exact: true })).toBeVisible();
+    await toolbar.getByRole("link", { name: "View errors", exact: true }).click();
+    await expect(page).toHaveURL(/\/tab\/errors/);
+    await expect(page).toHaveURL(/var-browser=Safari/);
+    await toolbar.getByRole("link", { name: "View sessions", exact: true }).click();
+    await expect(page).toHaveURL(/viz_type=sessions/);
+    expect(new URL(page.url()).searchParams.get("query")).toContain("Safari/");
+    expect(errors).toEqual([]);
+  } finally { cleanup(); }
+});

monoscope.cabal

@@ -269,6 +269,7 @@ extra-source-files:
     static/migrations/0252_project_memory_page_identity.sql
     static/migrations/0253_project_memory_legacy_writes.sql
     static/migrations/0254_ingest_sampling.sql
+    static/migrations/0254_rum_segments.sql
     test/fixtures/github-app-test.pem
     test/fixtures/wiki-browser.mjs
     static/public/assets/web-components/dist/manifest.json
@@ -380,6 +381,7 @@ library
       Pages.Projects
       Pages.Replay
       Pages.Reports
+      Pages.Rum
       Pages.ServiceMap
       Pages.Settings
       Pages.Share
@@ -412,6 +414,7 @@ library
       Pkg.Prometheus
       Pkg.QueryCache
       Pkg.Queue
+      Pkg.Rum
       Pkg.Sampling
       Pkg.SchemaLearning.Catalog
       Pkg.SchemaLearning.Hot
@@ -894,6 +897,7 @@ test-suite integration-tests
     , hs-opentelemetry-instrumentation-auto
     , hs-opentelemetry-instrumentation-hasql
     , hspec
+    , http-api-data
     , http-client
     , http-client-tls
     , http-types
@@ -1024,6 +1028,7 @@ test-suite test-dev
       Pages.Projects
       Pages.Replay
       Pages.Reports
+      Pages.Rum
       Pages.ServiceMap
       Pages.Settings
       Pages.Share
@@ -1056,6 +1061,7 @@ test-suite test-dev
       Pkg.Prometheus
       Pkg.QueryCache
       Pkg.Queue
+      Pkg.Rum
       Pkg.Sampling
       Pkg.SchemaLearning.Catalog
       Pkg.SchemaLearning.Hot

package.yaml

@@ -296,6 +296,7 @@ tests:
       - unix
       - lucid
       - http-types
+      - http-api-data
       - wai
       - wai-extra
       - unordered-containers

scripts/ci/ci.sh

@@ -332,9 +332,17 @@ CABAL_OPTS="--ghc-options=-O0 -j$CI_CORES +RTS -A64m -n2m -RTS"
 CABAL_FLAGS='-f-devtest'
 
 prepare_browser() {
+  [ "${CI_BROWSER_PREPARED:-}" != true ] || return 0
   (cd e2e && npm ci --prefer-offline --no-audit && \
     { [ -e /usr/local/share/monoscope-playwright-deps ] || npx playwright install-deps chromium; } && \
     npx playwright install chromium)
+  export CI_BROWSER_PREPARED=true
+}
+
+prepare_web_deps() {
+  [ "${CI_WEB_DEPS_PREPARED:-}" != true ] || return 0
+  (cd web-components && npm ci --prefer-offline --no-audit)
+  export CI_WEB_DEPS_PREPARED=true
 }
 
 run_body() { # <check>
@@ -384,7 +392,7 @@ run_body() { # <check>
       ;;
     release-tests) node --test scripts/ci/release-test.mjs ;;
     hlint)   hlint -j src/ shared/src cli ;;
-    ui-tests) (cd web-components && npm ci --prefer-offline --no-audit && npm test) ;;
+    ui-tests) prepare_web_deps; (cd web-components && npm test) ;;
     # Drives the real server in a real browser. scripts/e2e.sh starts that server itself on
     # 8081 against a throwaway database, so this only has to supply the binary and chromium.
     e2e)
@@ -410,7 +418,7 @@ run_integration() {
   # reached under CI_ALLOW_DEGRADED — cmd_run refuses this check otherwise.
   case " ${CAPS:-} " in *" tf-real "*) ;; *) unset TIMEFUSION_PG_TEST_URL ;; esac
   export USE_EXTERNAL_DB=true LOG_LEVEL=${LOG_LEVEL:-warn}
-  (cd web-components && npm ci --prefer-offline --no-audit)
+  prepare_web_deps
   prepare_browser
   make build-chart-cli
   [ -n "${CI_INTEGRATION_BIN:-}" ] || cabal build integration-tests -j $CABAL_FLAGS "$CABAL_OPTS"
@@ -484,6 +492,11 @@ stop_early() { [ "$1" -ne 0 ] && [ "${CI_KEEP_GOING:-}" != true ]; }
 
 # Background lanes live in $BG_DIR as <check>.{pid,log,result}.
 start_background() { # <check>
+  # npm ci removes node_modules: finish installs before any parallel consumer.
+  case "$1" in
+    integration-tests) prepare_web_deps; prepare_browser ;;
+    e2e) prepare_browser ;;
+  esac
   CI_ATTEST_OUT="$BG_DIR/$1.result" "$0" run "$1" > "$BG_DIR/$1.log" 2>&1 &
   echo $! > "$BG_DIR/$1.pid"
 }
@@ -1013,6 +1026,21 @@ cmd_selftest() {
   assert "background failure propagates" 1 "$body_rc"
   assert "background diagnostics retained" yes "$([ -f "$BG_DIR/failing.log" ] && echo yes || echo no)"
   rm -rf "$BG_DIR"
+  BG_DIR=$(mktemp -d -t ci-selftest-browser.XXXXXX)
+  printf '#!/bin/sh\necho install >> "%s/calls"\n' "$BG_DIR" > "$BG_DIR/npm"
+  printf '#!/bin/sh\nexit 0\n' > "$BG_DIR/npx"
+  chmod +x "$BG_DIR/npm" "$BG_DIR/npx"
+  (
+    export PATH="$BG_DIR:$PATH"
+    unset CI_BROWSER_PREPARED CI_WEB_DEPS_PREPARED
+    prepare_browser
+    prepare_web_deps
+    prepare_browser & browser_one=$!
+    prepare_web_deps & browser_two=$!
+    wait "$browser_one" "$browser_two"
+  )
+  assert "parallel lanes install each dependency tree once" 2 "$(wc -l < "$BG_DIR/calls" | tr -d ' ')"
+  rm -rf "$BG_DIR"
   # kafka-topics.sh under macOS /bin/bash 3.2 with no SASL user: an empty array under
   # `set -u` killed it silently (2026-10-20). A stub rpk records what it was asked to do.
   # Only bash < 4.4 reproduces the bug, so on Linux CI this checks the happy path alone.

src/Pages/Charts/Charts.hs

@@ -183,7 +183,7 @@ runQueryAST authCtx dbSource respDataType pid source binDensity environment serv
 
   case querySQLM of
     Just querySQL
-      | Just message <- rawSqlScopeError environment service querySQL ->
+      | Just message <- rawSqlScopeError environment service (fillQueryTemplate mappngSQL querySQL) ->
           pure (emptyMetricsFor now fromD toD){error = Just message}
     Just querySQL -> do
       let tbl = sourceTable source
@@ -223,7 +223,7 @@ runRawQuery authCtx dbSource decoder cfg ast kql template mapping liveRange prog
       (from, to) = bounds
       canonical =
         M.fromList [("query_ast_filters", rawAstFilters cacheCfg components)]
-          <> variablePresets cfg.binDensity cfg.pid.toText from to [] cfg.currentTime
+          <> variablePresets cfg.binDensity cfg.pid.toText from to [(key, Just value) | (key, value) <- M.toList mapping, "var-" `T.isPrefixOf` key || key == "table-search"] cfg.currentTime
           <> mapping
       sql = fillQueryTemplate canonical template
       fetch = liftIO $ fetchMetricsDataWithProgress progress decoder sql cfg.currentTime from to authCtx dbSource >>= either throwIO pure
@@ -755,7 +755,7 @@ queryMetricsStream dbSource dataTypeM pidM queryM querySQLM sinceM fromM toM sou
         maybe
           (first queryErrorText $ parseQueryDiagnosed source $ fillQueryTemplate mappingKQL $ maybeToMonoid $ Utils.nonEmptyT queryM)
           Left
-          (raw >>= rawSqlScopeError environment service)
+          (raw >>= rawSqlScopeError environment service . fillQueryTemplate mapping)
       scope = mkScopedQuery pid (fromD, toD) environment service
       cfg = (applyScopedQuery scope $ defSqlQueryCfg pid now source Nothing){binDensity = density, targetsTimefusion = usesTimefusionBackend authCtx.env.enableTimefusionReads dbSource}
       run emit = case parsed of