diff --git a/.gitignore b/.gitignore index 3271968..ef80b68 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ logs/ # Service runtime archives services/*/data/ data/ojo/ +.codex \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 6900e63..3d12444 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,36 @@ All notable changes to this project will be documented in this file. +## [0.0.4] - 2026-04-25 + +### Changed + +- Added a Windows-specific default config for `ojo-syslog` (`services/syslog/syslog.windows.yaml`) so Windows uses ETW-oriented defaults while Linux continues to use `services/syslog/syslog.yaml`. +- Updated `ojo-systrace` to namespace Windows-derived trace components as `windows.*` instead of reusing Linux `kernel.*` names, keeping the Windows trace shape compatible without sharing Linux naming. + +#### Efficient Archiving + +- Archives are now significantly more compressed and lightweight. Each service can be independently tuned via the YAML config, with full control over format, compression, retention, and archival strategy. + +```yaml +storage: + archive_enabled: true + archive_dir: data/ojo + archive_format: parquet # columnar storage for efficient reads + archive_compression: zstd # high-ratio compression + archive_mode: trend # archive strategy (`trend`, `lossless`, `forensic`) + archive_window_secs: 60 # rolling window per archive cycle + archive_max_file_bytes: 67108864 # 64 MiB per file + archive_retain_files: 8 # number of archive files to retain + archive_file_stem: ojo-snapshots +``` + +### Fixed + +- Fixed trend aggregation first-sample accounting in `host-collectors` so the first point in a window is no longer double-counted. +- Fixed parquet archive persistence in `host-collectors` so repeated writes preserve previously written active-file batches instead of truncating earlier archive data. + + ## [0.0.3] - 2026-04-15 ### Added diff --git a/Cargo.toml b/Cargo.toml index c55b8d5..cde16db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "ojo" version = "0.0.3" edition = "2021" +default-run = "ojo" [workspace] resolver = "2" @@ -27,6 +28,10 @@ ctrlc = "3" serde = { version = "1", features = ["derive"] } serde_yaml = "0.9" serde_json = "1" +arrow-array = "58" +parquet = { version = "58", default-features = false, features = ["arrow"] } +prost = "0.13" +snap = "1" tokio = { version = "1", features = [ "macros", "rt-multi-thread", diff --git a/README.md b/README.md index 4ee02e9..24e61da 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ These binaries are separate workspace crates under `services//`. Each runs | Systrace | `ojo-systrace` | `services/systrace/systrace.yaml` | `system.systrace.*` | | Syslog | `ojo-syslog` | `services/syslog/syslog.yaml` | `system.syslog.*` | -Shared OTLP and filtering helpers live in `crates/host-collectors`. Grafana dashboards for each extension are under `grafana/` (`docker.json`, `gpu.json`, `sensors.json`, `mysql.json`, `postgres.json`, `nfs-client.json`, `nginx.json`, `redis.json`, `systemd.json`, `systrace.json`, `syslog.json`). +Shared OTLP, filtering, and archive helpers live in [`crates/host-collectors`](crates/host-collectors/README.md). Grafana dashboards for each extension are under `grafana/` (`docker.json`, `gpu.json`, `sensors.json`, `mysql.json`, `postgres.json`, `nfs-client.json`, `nginx.json`, `redis.json`, `systemd.json`, `systrace.json`, `syslog.json`). Release archives include the main `ojo` agent and all sidecar services for Linux (`-unix`) and Windows (`-win`). @@ -258,6 +258,43 @@ cd services/mysql && cargo run -- --config mysql.yaml cd services/postgres && cargo run -- --config postgres.yaml cd services/nfs-client && cargo run -- --config nfs-client.yaml cd services/nginx && cargo run -- --config nginx.yaml + +## Archive Storage Modes + +Archive behavior is configured in each YAML `storage` section: + +- `archive_enabled` +- `archive_dir` +- `archive_max_file_bytes` +- `archive_retain_files` +- `archive_file_stem` +- `archive_format` (`parquet`) +- `archive_mode` (`trend`, `lossless`, `forensic`) +- `archive_window_secs` (trend mode window) +- `archive_compression` (`zstd`) + +Mode guidance: +- `trend`: lossy, very compact, best for long-range dashboards and baselines. +- `lossless`: full-fidelity records with efficient parquet+zstd compression. +- `forensic`: compatibility row mode for deeper incident-level archives. + +## Archive Replay + +Replay archived trend parquet files to OTLP or Prometheus remote-write (Mimir compatible): + +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/sensors/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/sensors/data \ + --endpoint http://localhost:4320/mimir/api/v1/push \ + --protocol remote-write +``` cd services/redis && cargo run -- --config redis.yaml cd services/systemd && cargo run -- --config systemd.yaml cd services/systrace && cargo run -- --config systrace.yaml @@ -399,3 +436,10 @@ Cross-checking to `x86_64-pc-windows-gnu` from Linux requires a MinGW toolchain - Ensure `collection.include_process_metrics: true` - Verify process permissions - Check that `metrics` filters are not excluding `process.` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/crates/host-collectors/Cargo.toml b/crates/host-collectors/Cargo.toml index f2c1cfa..736089e 100644 --- a/crates/host-collectors/Cargo.toml +++ b/crates/host-collectors/Cargo.toml @@ -7,6 +7,9 @@ edition = "2021" anyhow = "1" gethostname = "1" serde_json = "1" +arrow-array = "58" +arrow-schema = "58" +parquet = { version = "58", default-features = false, features = ["arrow", "zstd"] } opentelemetry = { version = "0.31", features = ["metrics", "trace"] } opentelemetry_sdk = { version = "0.31", features = ["metrics", "trace", "rt-tokio", "experimental_trace_batch_span_processor_with_async_runtime"] } opentelemetry-otlp = { version = "0.31", features = [ diff --git a/crates/host-collectors/README.md b/crates/host-collectors/README.md new file mode 100644 index 0000000..ffae1ff --- /dev/null +++ b/crates/host-collectors/README.md @@ -0,0 +1,57 @@ +# Host Collectors + +Shared runtime library used by `ojo` and all sidecar services. + +## What it provides +- OTLP meter/tracer provider setup helpers. +- Prefix-based metric filtering helpers. +- Unified archive writer implementation used across services. + +## Archive configuration contract +The shared archive config is modeled by `ArchiveStorageConfig` and consumed by all services: + +- `archive_enabled` +- `archive_dir` +- `archive_max_file_bytes` +- `archive_retain_files` +- `archive_file_stem` +- `archive_format` (currently `parquet`) +- `archive_mode` (`trend`, `lossless`, `forensic`) +- `archive_window_secs` (trend mode) +- `archive_compression` (`zstd`) + +## Archive modes +- `trend`: compact lossy summaries by time window (`min/max/avg/count/first/last`). +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode with row-level payload retention. + +## Typical outputs +- trend: `-trend.parquet` +- lossless: `-lossless.parquet` +- forensic: `-forensic.parquet` + +## Replay +Use the workspace replay tool: + +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + +Remote-write example: + +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --endpoint http://localhost:4320/mimir/api/v1/push \ + --protocol remote-write +``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/crates/host-collectors/src/lib.rs b/crates/host-collectors/src/lib.rs index bd605a4..a816bab 100644 --- a/crates/host-collectors/src/lib.rs +++ b/crates/host-collectors/src/lib.rs @@ -8,15 +8,100 @@ use opentelemetry_sdk::{ resource::Resource, trace::{BatchConfigBuilder, BatchSpanProcessor, SdkTracerProvider, SpanData, SpanExporter}, }; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use parquet::arrow::arrow_writer::ArrowWriter; +use parquet::basic::{Compression, ZstdLevel}; +use parquet::file::properties::WriterProperties; use serde_json::Value; -use std::collections::BTreeMap; -use std::fs::{self, OpenOptions}; -use std::io::Write; +use std::collections::{BTreeMap, HashMap}; +use std::fs::{self, File}; +use std::hash::{Hash, Hasher}; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; -use std::time::Duration; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use arrow_array::{ArrayRef, Float64Array, Int64Array, RecordBatch, StringArray, UInt64Array}; +use arrow_schema::{DataType, Field, Schema}; +use std::sync::Arc; pub const METRIC_PREFIX_SYSTEM: &str = "system."; +const ARCHIVE_SCHEMA_VERSION: &str = "v1"; +const DEFAULT_ARCHIVE_WINDOW_SECS: u64 = 60; +const DEFAULT_TOP_K_LOG_SIGNATURES: usize = 3; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum ArchiveFormat { + #[default] + Parquet, +} + +impl ArchiveFormat { + pub fn parse(value: Option<&str>) -> Self { + match value.map(|v| v.trim().to_ascii_lowercase()) { + Some(v) if v == "parquet" || v.is_empty() => Self::Parquet, + _ => Self::Parquet, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Parquet => "parquet", + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum ArchiveMode { + #[default] + Trend, + Forensic, + Lossless, +} + +impl ArchiveMode { + pub fn parse(value: Option<&str>) -> Self { + match value.map(|v| v.trim().to_ascii_lowercase()) { + Some(v) if v == "forensic" => Self::Forensic, + Some(v) if v == "lossless" => Self::Lossless, + _ => Self::Trend, + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Trend => "trend", + Self::Forensic => "forensic", + Self::Lossless => "lossless", + } + } +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub enum ArchiveCompression { + #[default] + Zstd, +} + +impl ArchiveCompression { + pub fn parse(value: Option<&str>) -> Self { + match value.map(|v| v.trim().to_ascii_lowercase()) { + Some(v) if v == "zstd" || v.is_empty() => Self::Zstd, + _ => Self::Zstd, + } + } + + fn to_parquet(&self) -> Compression { + match self { + Self::Zstd => Compression::ZSTD(ZstdLevel::try_new(3).unwrap_or_default()), + } + } + + pub fn as_str(&self) -> &'static str { + match self { + Self::Zstd => "zstd", + } + } +} #[derive(Clone, Debug)] pub struct ArchiveStorageConfig { @@ -25,6 +110,10 @@ pub struct ArchiveStorageConfig { pub max_file_bytes: u64, pub retain_files: usize, pub file_stem: String, + pub format: ArchiveFormat, + pub mode: ArchiveMode, + pub window_secs: u64, + pub compression: ArchiveCompression, } impl ArchiveStorageConfig { @@ -35,24 +124,129 @@ impl ArchiveStorageConfig { max_file_bytes: 0, retain_files: 0, file_stem: default_stem.to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: DEFAULT_ARCHIVE_WINDOW_SECS, + compression: ArchiveCompression::Zstd, + } + } +} + +pub trait ArchiveWriter { + fn write_snapshot(&mut self, value: &Value); + fn write_log_batch(&mut self, values: &[Value]); + fn is_healthy(&self) -> bool; + fn total_records(&self) -> u64; + fn total_bytes(&self) -> u64; + fn last_error(&self) -> Option<&str>; +} + +#[derive(Clone, Debug)] +struct TrendStats { + min: f64, + max: f64, + sum: f64, + count: u64, + first: f64, + last: f64, +} + +impl TrendStats { + fn new(value: f64) -> Self { + Self { + min: value, + max: value, + sum: value, + count: 1, + first: value, + last: value, } } + + fn update(&mut self, value: f64) { + if value < self.min { + self.min = value; + } + if value > self.max { + self.max = value; + } + self.sum += value; + self.count = self.count.saturating_add(1); + self.last = value; + } + + fn avg(&self) -> f64 { + if self.count == 0 { + 0.0 + } else { + self.sum / self.count as f64 + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +struct TrendKey { + service_name: String, + instance_id: String, + metric_key: String, +} + +impl Hash for TrendKey { + fn hash(&self, state: &mut H) { + self.service_name.hash(state); + self.instance_id.hash(state); + self.metric_key.hash(state); + } +} + +#[derive(Clone, Debug)] +struct TrendRow { + window_start_unix_secs: i64, + service_name: String, + instance_id: String, + metric_key: String, + min: f64, + max: f64, + avg: f64, + count: u64, + first: f64, + last: f64, + written_at_unix_ms: i64, } #[derive(Clone, Debug)] -pub struct JsonArchiveWriter { +struct ForensicRow { + observed_unix_ms: i64, + service_name: String, + instance_id: String, + record_type: String, + severity: Option, + source: Option, + watch_target: Option, + message_signature: Option, + sample_message: Option, + payload_json: String, +} + +pub struct ParquetArchiveWriter { enabled: bool, dir: String, max_file_bytes: u64, retain_files: usize, file_stem: String, + mode: ArchiveMode, + window_secs: u64, + compression: ArchiveCompression, pub total_records: u64, pub total_bytes: u64, pub healthy: bool, pub last_error: Option, + default_service_name: String, + default_instance_id: String, + trend_windows: BTreeMap>, } -impl JsonArchiveWriter { +impl ParquetArchiveWriter { pub fn from_config(config: &ArchiveStorageConfig) -> Self { Self { enabled: config.enabled, @@ -60,44 +254,502 @@ impl JsonArchiveWriter { max_file_bytes: config.max_file_bytes, retain_files: config.retain_files, file_stem: config.file_stem.clone(), + mode: config.mode.clone(), + window_secs: config.window_secs.max(1), + compression: config.compression.clone(), total_records: 0, total_bytes: 0, healthy: true, last_error: None, + default_service_name: String::new(), + default_instance_id: String::new(), + trend_windows: BTreeMap::new(), } } + pub fn set_default_identity(&mut self, service_name: &str, instance_id: &str) { + self.default_service_name = service_name.to_string(); + self.default_instance_id = instance_id.to_string(); + } + pub fn write_json_line(&mut self, value: &Value) { + self.write_snapshot(value); + } + + pub fn flush(&mut self) { if !self.enabled { return; } - if let Err(err) = self.write_json_line_impl(value) { - self.healthy = false; - self.last_error = Some(err.to_string()); - } else { - self.healthy = true; - self.last_error = None; + if let ArchiveMode::Trend = self.mode { + if let Err(err) = self.flush_all_trend_windows() { + self.healthy = false; + self.last_error = Some(err.to_string()); + } } } - fn write_json_line_impl(&mut self, value: &Value) -> anyhow::Result<()> { + fn write_snapshot_impl(&mut self, value: &Value) -> anyhow::Result<()> { fs::create_dir_all(&self.dir)?; - let path = format!("{}/{}.ndjson", self.dir, self.file_stem); - self.rotate_if_needed(&path)?; - - let mut file = OpenOptions::new().create(true).append(true).open(&path)?; - let line = value.to_string(); - file.write_all(line.as_bytes())?; - file.write_all(b"\n")?; self.total_records = self.total_records.saturating_add(1); self.total_bytes = self .total_bytes - .saturating_add((line.len() as u64).saturating_add(1)); + .saturating_add((value.to_string().len() as u64).saturating_add(1)); + + match self.mode { + ArchiveMode::Trend => self.ingest_snapshot_to_trend(value), + ArchiveMode::Forensic | ArchiveMode::Lossless => { + self.write_forensic_rows(vec![self.build_forensic_row("snapshot", value)]) + } + } + } + + fn write_log_batch_impl(&mut self, values: &[Value]) -> anyhow::Result<()> { + if values.is_empty() { + return Ok(()); + } + + fs::create_dir_all(&self.dir)?; + self.total_records = self.total_records.saturating_add(values.len() as u64); + self.total_bytes = self.total_bytes.saturating_add( + values + .iter() + .map(|v| v.to_string().len() as u64 + 1) + .sum::(), + ); + match self.mode { + ArchiveMode::Trend => self.ingest_logs_to_trend(values), + ArchiveMode::Forensic | ArchiveMode::Lossless => { + let rows = values + .iter() + .map(|v| self.build_forensic_row("log", v)) + .collect::>(); + self.write_forensic_rows(rows) + } + } + } + + fn ingest_snapshot_to_trend(&mut self, value: &Value) -> anyhow::Result<()> { + let now_secs = now_unix_secs(); + let window_start = align_window_start(now_secs, self.window_secs); + let (service_name, instance_id) = self.resolve_identity(value); + + if let Some(obj) = value.as_object() { + for (k, v) in obj { + let mut path = vec![k.to_string()]; + self.flatten_numeric(v, &mut path, window_start, &service_name, &instance_id); + } + } + + self.flush_closed_windows(window_start) + } + + fn ingest_logs_to_trend(&mut self, values: &[Value]) -> anyhow::Result<()> { + let now_secs = now_unix_secs(); + let window_start = align_window_start(now_secs, self.window_secs); + + for value in values { + let (service_name, instance_id) = self.resolve_identity(value); + self.add_trend_point( + window_start, + TrendKey { + service_name: service_name.clone(), + instance_id: instance_id.clone(), + metric_key: "log.count".to_string(), + }, + 1.0, + ); + + if let Some(severity) = value.get("severity_text").and_then(Value::as_str) { + self.add_trend_point( + window_start, + TrendKey { + service_name: service_name.clone(), + instance_id: instance_id.clone(), + metric_key: format!("log.severity.{}.count", sanitize_segment(severity)), + }, + 1.0, + ); + } + if let Some(source) = value.get("source").and_then(Value::as_str) { + self.add_trend_point( + window_start, + TrendKey { + service_name: service_name.clone(), + instance_id: instance_id.clone(), + metric_key: format!("log.source.{}.count", sanitize_segment(source)), + }, + 1.0, + ); + } + if let Some(target) = value.get("watch_target").and_then(Value::as_str) { + self.add_trend_point( + window_start, + TrendKey { + service_name: service_name.clone(), + instance_id: instance_id.clone(), + metric_key: format!("log.watch_target.{}.count", sanitize_segment(target)), + }, + 1.0, + ); + } + } + + let mut signatures = HashMap::<(String, String, u64), (u64, String)>::new(); + for value in values { + if let Some(body) = value.get("body").and_then(Value::as_str) { + let (service_name, instance_id) = self.resolve_identity(value); + let sig = stable_signature(body); + let entry = signatures + .entry((service_name, instance_id, sig)) + .or_insert((0, body.to_string())); + entry.0 = entry.0.saturating_add(1); + } + } + + let mut ranked = signatures.into_iter().collect::>(); + ranked.sort_by_key(|entry| std::cmp::Reverse(entry.1 .0)); + for ((service_name, instance_id, sig), (count, _sample)) in + ranked.into_iter().take(DEFAULT_TOP_K_LOG_SIGNATURES) + { + let metric_key = format!("log.topk.signature.{}.count", sig); + self.add_trend_point( + window_start, + TrendKey { + service_name, + instance_id, + metric_key, + }, + count as f64, + ); + } + + self.flush_closed_windows(window_start) + } + + fn flatten_numeric( + &mut self, + value: &Value, + path: &mut Vec, + window_start: i64, + service_name: &str, + instance_id: &str, + ) { + match value { + Value::Number(num) => { + if let Some(v) = num.as_f64().filter(|candidate| candidate.is_finite()) { + self.add_trend_point( + window_start, + TrendKey { + service_name: service_name.to_string(), + instance_id: instance_id.to_string(), + metric_key: path.join("."), + }, + v, + ); + } + } + Value::Object(map) => { + for (k, v) in map { + path.push(k.to_string()); + self.flatten_numeric(v, path, window_start, service_name, instance_id); + path.pop(); + } + } + Value::Array(values) => { + for (idx, v) in values.iter().enumerate() { + path.push(idx.to_string()); + self.flatten_numeric(v, path, window_start, service_name, instance_id); + path.pop(); + } + } + _ => {} + } + } + + fn add_trend_point(&mut self, window_start: i64, key: TrendKey, value: f64) { + let map = self.trend_windows.entry(window_start).or_default(); + match map.entry(key) { + std::collections::hash_map::Entry::Occupied(mut existing) => { + existing.get_mut().update(value); + } + std::collections::hash_map::Entry::Vacant(vacant) => { + vacant.insert(TrendStats::new(value)); + } + } + } + + fn flush_closed_windows(&mut self, current_window: i64) -> anyhow::Result<()> { + let closed = self + .trend_windows + .keys() + .copied() + .filter(|w| *w < current_window) + .collect::>(); + for window in closed { + let values = self + .trend_windows + .remove(&window) + .expect("closed window key collected from map must exist"); + self.write_trend_window(window, values)?; + } + Ok(()) + } + + fn flush_all_trend_windows(&mut self) -> anyhow::Result<()> { + let windows = self.trend_windows.keys().copied().collect::>(); + for window in windows { + let values = self + .trend_windows + .remove(&window) + .expect("window key collected from map must exist"); + self.write_trend_window(window, values)?; + } + Ok(()) + } + + fn write_trend_window( + &mut self, + window_start: i64, + values: HashMap, + ) -> anyhow::Result<()> { + if values.is_empty() { + return Ok(()); + } + + let written_at = now_unix_millis(); + let rows = values + .into_iter() + .map(|(key, stats)| TrendRow { + window_start_unix_secs: window_start, + service_name: key.service_name, + instance_id: key.instance_id, + metric_key: key.metric_key, + min: stats.min, + max: stats.max, + avg: stats.avg(), + count: stats.count, + first: stats.first, + last: stats.last, + written_at_unix_ms: written_at, + }) + .collect::>(); + + self.write_trend_rows(rows) + } + + fn write_trend_rows(&mut self, rows: Vec) -> anyhow::Result<()> { + let schema = trend_schema(); + + let columns: Vec = vec![ + Arc::new(Int64Array::from( + rows.iter() + .map(|r| r.window_start_unix_secs) + .collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.service_name.as_str()) + .collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.instance_id.as_str()) + .collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.metric_key.as_str()) + .collect::>(), + )), + Arc::new(Float64Array::from( + rows.iter().map(|r| r.min).collect::>(), + )), + Arc::new(Float64Array::from( + rows.iter().map(|r| r.max).collect::>(), + )), + Arc::new(Float64Array::from( + rows.iter().map(|r| r.avg).collect::>(), + )), + Arc::new(UInt64Array::from( + rows.iter().map(|r| r.count).collect::>(), + )), + Arc::new(Float64Array::from( + rows.iter().map(|r| r.first).collect::>(), + )), + Arc::new(Float64Array::from( + rows.iter().map(|r| r.last).collect::>(), + )), + Arc::new(StringArray::from(vec![ARCHIVE_SCHEMA_VERSION; rows.len()])), + Arc::new(StringArray::from(vec![self.mode.as_str(); rows.len()])), + Arc::new(Int64Array::from( + rows.iter() + .map(|r| r.written_at_unix_ms) + .collect::>(), + )), + ]; + + let batch = RecordBatch::try_new(schema, columns)?; + self.write_batch(batch, "trend") + } + + fn build_forensic_row(&self, record_type: &str, value: &Value) -> ForensicRow { + let (service_name, instance_id) = self.resolve_identity(value); + let body = value + .get("body") + .and_then(Value::as_str) + .map(ToOwned::to_owned); + + ForensicRow { + observed_unix_ms: now_unix_millis(), + service_name, + instance_id, + record_type: record_type.to_string(), + severity: value + .get("severity_text") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + source: value + .get("source") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + watch_target: value + .get("watch_target") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + message_signature: body.as_ref().map(|s| stable_signature(s)), + sample_message: body, + payload_json: value.to_string(), + } + } + + fn write_forensic_rows(&mut self, rows: Vec) -> anyhow::Result<()> { + if rows.is_empty() { + return Ok(()); + } + let schema = forensic_schema(); + + let columns: Vec = vec![ + Arc::new(Int64Array::from( + rows.iter().map(|r| r.observed_unix_ms).collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.service_name.as_str()) + .collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.instance_id.as_str()) + .collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.record_type.as_str()) + .collect::>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.severity.as_deref()) + .collect::>>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.source.as_deref()) + .collect::>>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.watch_target.as_deref()) + .collect::>>(), + )), + Arc::new(UInt64Array::from( + rows.iter() + .map(|r| r.message_signature) + .collect::>>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.sample_message.as_deref()) + .collect::>>(), + )), + Arc::new(StringArray::from( + rows.iter() + .map(|r| r.payload_json.as_str()) + .collect::>(), + )), + Arc::new(StringArray::from(vec![ARCHIVE_SCHEMA_VERSION; rows.len()])), + Arc::new(StringArray::from(vec![self.mode.as_str(); rows.len()])), + ]; + + let batch = RecordBatch::try_new(schema, columns)?; + let suffix = match self.mode { + ArchiveMode::Lossless => "lossless", + _ => "forensic", + }; + self.write_batch(batch, suffix) + } + + fn resolve_identity(&self, value: &Value) -> (String, String) { + let (service_name, instance_id) = extract_identity(value); + let resolved_service = if service_name.is_empty() { + self.default_service_name.clone() + } else { + service_name + }; + let resolved_instance = if instance_id.is_empty() { + self.default_instance_id.clone() + } else { + instance_id + }; + (resolved_service, resolved_instance) + } + + fn write_batch(&mut self, batch: RecordBatch, suffix: &str) -> anyhow::Result<()> { + let path = format!("{}/{}-{}.parquet", self.dir, self.file_stem, suffix); + self.rotate_if_needed(&path)?; + let existing_batches = self.read_existing_batches(&path, batch.schema())?; + + let mut metadata_builder = WriterProperties::builder(); + metadata_builder = metadata_builder + .set_compression(self.compression.to_parquet()) + .set_created_by("ojo-archive".to_string()); + + let props = metadata_builder.build(); + let file = File::create(&path)?; + let mut writer = ArrowWriter::try_new(file, batch.schema(), Some(props))?; + for existing in existing_batches { + writer.write(&existing)?; + } + writer.write(&batch)?; + writer.flush()?; + writer.finish()?; self.prune_rotated_files(&path)?; Ok(()) } + fn read_existing_batches( + &self, + path: &str, + expected_schema: Arc, + ) -> anyhow::Result> { + if !Path::new(path).exists() { + return Ok(Vec::new()); + } + + let file = File::open(path)?; + let mut reader = ParquetRecordBatchReaderBuilder::try_new(file)?.build()?; + let mut batches = Vec::new(); + for maybe_batch in &mut reader { + let batch = maybe_batch?; + if batch.schema().as_ref() != expected_schema.as_ref() { + anyhow::bail!("existing archive schema mismatch for path: {path}"); + } + batches.push(batch); + } + Ok(batches) + } + fn rotate_if_needed(&self, path: &str) -> anyhow::Result<()> { let metadata = match fs::metadata(path) { Ok(md) => md, @@ -106,6 +758,7 @@ impl JsonArchiveWriter { if metadata.len() < self.max_file_bytes { return Ok(()); } + for idx in (1..=self.retain_files).rev() { let src = format!("{}.{}", path, idx); let dst = format!("{}.{}", path, idx + 1); @@ -134,6 +787,164 @@ impl JsonArchiveWriter { } } +impl ArchiveWriter for ParquetArchiveWriter { + fn write_snapshot(&mut self, value: &Value) { + if !self.enabled { + return; + } + if let Err(err) = self.write_snapshot_impl(value) { + self.healthy = false; + self.last_error = Some(err.to_string()); + } else { + self.healthy = true; + self.last_error = None; + } + } + + fn write_log_batch(&mut self, values: &[Value]) { + if !self.enabled { + return; + } + if let Err(err) = self.write_log_batch_impl(values) { + self.healthy = false; + self.last_error = Some(err.to_string()); + } else { + self.healthy = true; + self.last_error = None; + } + } + + fn is_healthy(&self) -> bool { + self.healthy + } + + fn total_records(&self) -> u64 { + self.total_records + } + + fn total_bytes(&self) -> u64 { + self.total_bytes + } + + fn last_error(&self) -> Option<&str> { + self.last_error.as_deref() + } +} + +impl Drop for ParquetArchiveWriter { + fn drop(&mut self) { + self.flush(); + } +} + +pub type JsonArchiveWriter = ParquetArchiveWriter; + +fn trend_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("window_start_unix_secs", DataType::Int64, false), + Field::new("service_name", DataType::Utf8, false), + Field::new("instance_id", DataType::Utf8, false), + Field::new("metric_key", DataType::Utf8, false), + Field::new("min", DataType::Float64, false), + Field::new("max", DataType::Float64, false), + Field::new("avg", DataType::Float64, false), + Field::new("count", DataType::UInt64, false), + Field::new("first", DataType::Float64, false), + Field::new("last", DataType::Float64, false), + Field::new("archive_schema_version", DataType::Utf8, false), + Field::new("archive_mode", DataType::Utf8, false), + Field::new("written_at_unix_ms", DataType::Int64, false), + ])) +} + +fn forensic_schema() -> Arc { + Arc::new(Schema::new(vec![ + Field::new("observed_unix_ms", DataType::Int64, false), + Field::new("service_name", DataType::Utf8, false), + Field::new("instance_id", DataType::Utf8, false), + Field::new("record_type", DataType::Utf8, false), + Field::new("severity", DataType::Utf8, true), + Field::new("source", DataType::Utf8, true), + Field::new("watch_target", DataType::Utf8, true), + Field::new("message_signature", DataType::UInt64, true), + Field::new("sample_message", DataType::Utf8, true), + Field::new("payload_json", DataType::Utf8, false), + Field::new("archive_schema_version", DataType::Utf8, false), + Field::new("archive_mode", DataType::Utf8, false), + ])) +} + +fn sanitize_segment(raw: &str) -> String { + let lowered = raw.to_ascii_lowercase(); + let cleaned = lowered + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' { + ch + } else { + '_' + } + }) + .collect::(); + + cleaned.trim_matches('_').to_string() +} + +fn stable_signature(value: &str) -> u64 { + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + value.hash(&mut hasher); + hasher.finish() +} + +fn now_unix_secs() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() as i64 +} + +fn now_unix_millis() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() as i64 +} + +fn align_window_start(timestamp_secs: i64, window_secs: u64) -> i64 { + let ws = window_secs.max(1) as i64; + (timestamp_secs / ws) * ws +} + +fn extract_identity(value: &Value) -> (String, String) { + let service_name = value + .get("service_name") + .and_then(Value::as_str) + .or_else(|| { + value + .get("service") + .and_then(Value::as_object) + .and_then(|m| m.get("name")) + .and_then(Value::as_str) + }) + .unwrap_or_default() + .to_string(); + + let instance_id = value + .get("instance_id") + .and_then(Value::as_str) + .or_else(|| { + value + .get("service") + .and_then(Value::as_object) + .and_then(|m| m.get("instance_id")) + .and_then(Value::as_str) + }) + .unwrap_or_default() + .to_string(); + + (service_name, instance_id) +} + #[derive(Clone, Debug)] pub struct PrefixFilter { include: Vec, diff --git a/crates/host-collectors/src/tests.rs b/crates/host-collectors/src/tests.rs index 006196b..f450eeb 100644 --- a/crates/host-collectors/src/tests.rs +++ b/crates/host-collectors/src/tests.rs @@ -1,8 +1,10 @@ use super::{ build_meter_provider, build_tracer_provider, default_protocol_for_endpoint, hostname, - init_meter_provider, init_tracer_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, StdoutSpanExporter, METRIC_PREFIX_SYSTEM, + init_meter_provider, init_tracer_provider, ArchiveCompression, ArchiveFormat, ArchiveMode, + ArchiveStorageConfig, ArchiveWriter, JsonArchiveWriter, OtlpSettings, PrefixFilter, + StdoutSpanExporter, METRIC_PREFIX_SYSTEM, }; +use arrow_array::{Array, StringArray}; use opentelemetry::trace::{ SpanContext, SpanId, SpanKind, Status, TraceFlags, TraceId, TraceState, }; @@ -11,8 +13,12 @@ use opentelemetry::KeyValue; use opentelemetry_sdk::error::OTelSdkError; use opentelemetry_sdk::resource::Resource; use opentelemetry_sdk::trace::{SpanData, SpanEvents, SpanExporter, SpanLinks}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use serde_json::json; use std::collections::BTreeMap; +use std::collections::HashMap; use std::fs; +use std::fs::File; use std::path::PathBuf; use std::time::Duration; use std::time::{SystemTime, UNIX_EPOCH}; @@ -121,6 +127,13 @@ fn default_protocol_without_scheme_uses_grpc() { ); } +#[test] +fn archive_mode_parses_lossless_and_forensic() { + assert_eq!(ArchiveMode::parse(Some("lossless")), ArchiveMode::Lossless); + assert_eq!(ArchiveMode::parse(Some("forensic")), ArchiveMode::Forensic); + assert_eq!(ArchiveMode::parse(Some("trend")), ArchiveMode::Trend); +} + #[test] fn build_meter_provider_grpc_succeeds() { let rt = tokio::runtime::Runtime::new().expect("runtime"); @@ -279,6 +292,10 @@ fn json_archive_writer_from_config_copies_fields_and_initializes_state() { max_file_bytes: 128, retain_files: 3, file_stem: "host".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Forensic, + window_secs: 60, + compression: ArchiveCompression::Zstd, }; let writer = JsonArchiveWriter::from_config(&config); @@ -312,6 +329,10 @@ fn json_archive_writer_writes_and_tracks_bytes() { max_file_bytes: 1024, retain_files: 0, file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 60, + compression: ArchiveCompression::Zstd, }; let mut writer = JsonArchiveWriter::from_config(&config); let payload = serde_json::json!({"name": "cpu", "value": 1}); @@ -328,6 +349,56 @@ fn json_archive_writer_writes_and_tracks_bytes() { fs::remove_dir_all(&dir).expect("remove temp directory"); } +#[test] +fn json_archive_writer_uses_default_identity_when_missing_in_payload() { + let dir = unique_temp_dir("host_collectors_archive_identity_defaults"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 0, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 1, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + writer.set_default_identity("svc-default", "inst-default"); + writer.write_json_line(&serde_json::json!({"metric": {"value": 42}})); + writer.flush(); + + let path = dir.join("metrics-trend.parquet"); + let file = File::open(path).expect("open archive parquet"); + let mut reader = ParquetRecordBatchReaderBuilder::try_new(file) + .expect("build parquet reader") + .build() + .expect("create record batch reader"); + let batch = reader + .next() + .expect("at least one record batch") + .expect("read batch"); + let service_col = batch + .column_by_name("service_name") + .expect("service_name column") + .as_any() + .downcast_ref::() + .expect("service_name should be utf8"); + let instance_col = batch + .column_by_name("instance_id") + .expect("instance_id column") + .as_any() + .downcast_ref::() + .expect("instance_id should be utf8"); + + assert_eq!(service_col.value(0), "svc-default"); + assert_eq!(instance_col.value(0), "inst-default"); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + #[test] fn json_archive_writer_sets_health_false_when_archive_dir_is_invalid() { let dir = unique_temp_dir("host_collectors_archive_err"); @@ -341,6 +412,10 @@ fn json_archive_writer_sets_health_false_when_archive_dir_is_invalid() { max_file_bytes: 1024, retain_files: 1, file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 60, + compression: ArchiveCompression::Zstd, }; let mut writer = JsonArchiveWriter::from_config(&config); writer.write_json_line(&serde_json::json!({"name": "cpu"})); @@ -357,7 +432,7 @@ fn json_archive_writer_sets_health_false_when_archive_dir_is_invalid() { fn json_archive_writer_rotates_and_prunes_files() { let dir = unique_temp_dir("host_collectors_archive_rotate"); fs::create_dir_all(&dir).expect("create temp directory"); - let base = dir.join("metrics.ndjson"); + let base = dir.join("metrics-forensic.parquet"); fs::write(&base, vec![b'x'; 20]).expect("seed oversized current file"); fs::write(format!("{}.1", base.to_string_lossy()), b"older-1").expect("seed rotated file 1"); fs::write(format!("{}.2", base.to_string_lossy()), b"older-2").expect("seed rotated file 2"); @@ -369,6 +444,10 @@ fn json_archive_writer_rotates_and_prunes_files() { max_file_bytes: 10, retain_files: 2, file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 60, + compression: ArchiveCompression::Zstd, }; let mut writer = JsonArchiveWriter::from_config(&config); writer.write_json_line(&serde_json::json!({"rotated": true})); @@ -376,8 +455,6 @@ fn json_archive_writer_rotates_and_prunes_files() { assert!(base.exists()); assert!(PathBuf::from(format!("{}.1", base.to_string_lossy())).exists()); assert!(PathBuf::from(format!("{}.2", base.to_string_lossy())).exists()); - assert!(PathBuf::from(format!("{}.3", base.to_string_lossy())).exists()); - assert!(!PathBuf::from(format!("{}.4", base.to_string_lossy())).exists()); fs::remove_dir_all(&dir).expect("remove temp directory"); } @@ -420,3 +497,757 @@ fn build_tracer_provider_stdout_succeeds() { let result = build_tracer_provider(&settings); assert!(result.is_ok(), "stdout tracer builder: {result:?}"); } + +#[test] +fn archive_format_and_compression_parse_and_render() { + assert_eq!(ArchiveFormat::parse(None), ArchiveFormat::Parquet); + assert_eq!(ArchiveFormat::parse(Some("")), ArchiveFormat::Parquet); + assert_eq!( + ArchiveFormat::parse(Some("parquet")), + ArchiveFormat::Parquet + ); + assert_eq!( + ArchiveFormat::parse(Some("unknown")), + ArchiveFormat::Parquet + ); + assert_eq!(ArchiveFormat::Parquet.as_str(), "parquet"); + + assert_eq!(ArchiveCompression::parse(None), ArchiveCompression::Zstd); + assert_eq!( + ArchiveCompression::parse(Some("zstd")), + ArchiveCompression::Zstd + ); + assert_eq!( + ArchiveCompression::parse(Some("other")), + ArchiveCompression::Zstd + ); + assert_eq!(ArchiveCompression::Zstd.as_str(), "zstd"); + + assert_eq!(ArchiveMode::Trend.as_str(), "trend"); + assert_eq!(ArchiveMode::Forensic.as_str(), "forensic"); + assert_eq!(ArchiveMode::Lossless.as_str(), "lossless"); +} + +#[test] +fn archive_storage_config_disabled_covers_all_defaults() { + let disabled = ArchiveStorageConfig::disabled("metrics"); + assert!(!disabled.enabled); + assert_eq!(disabled.archive_dir, ""); + assert_eq!(disabled.max_file_bytes, 0); + assert_eq!(disabled.retain_files, 0); + assert_eq!(disabled.file_stem, "metrics"); + assert_eq!(disabled.format, ArchiveFormat::Parquet); + assert_eq!(disabled.mode, ArchiveMode::Trend); + assert_eq!(disabled.window_secs, 60); + assert_eq!(disabled.compression, ArchiveCompression::Zstd); +} + +#[test] +fn helper_functions_are_covered() { + assert_eq!( + super::sanitize_segment(" Error.Level / CPU% "), + "error_level___cpu" + ); + + let sig_a = super::stable_signature("same"); + let sig_b = super::stable_signature("same"); + let sig_c = super::stable_signature("different"); + assert_eq!(sig_a, sig_b); + assert_ne!(sig_a, sig_c); + + let secs = super::now_unix_secs(); + let millis = super::now_unix_millis(); + assert!(secs >= 0); + assert!(millis >= secs * 1000); + assert_eq!(super::align_window_start(125, 60), 120); + assert_eq!(super::align_window_start(125, 0), 125); + + let direct = json!({"service_name": "svc-a", "instance_id": "inst-a"}); + assert_eq!( + super::extract_identity(&direct), + ("svc-a".to_string(), "inst-a".to_string()) + ); + + let nested = json!({"service": {"name": "svc-b", "instance_id": "inst-b"}}); + assert_eq!( + super::extract_identity(&nested), + ("svc-b".to_string(), "inst-b".to_string()) + ); + + let missing = json!({"other": 1}); + assert_eq!( + super::extract_identity(&missing), + ("".to_string(), "".to_string()) + ); +} + +#[test] +fn build_providers_with_default_timeout_when_none() { + let rt = tokio::runtime::Runtime::new().expect("runtime"); + + let mut meter_settings = test_settings("grpc"); + meter_settings.otlp_timeout = None; + rt.block_on(async { + let meter_result = build_meter_provider(&meter_settings); + assert!( + meter_result.is_ok(), + "meter provider with default timeout: {meter_result:?}" + ); + }); + + let mut tracer_settings = test_settings("grpc"); + tracer_settings.otlp_timeout = None; + rt.block_on(async { + let tracer_result = build_tracer_provider(&tracer_settings); + assert!( + tracer_result.is_ok(), + "tracer provider with default timeout: {tracer_result:?}" + ); + }); +} + +#[test] +fn trend_mode_writes_log_metrics_and_snapshot_metrics() { + let dir = unique_temp_dir("host_collectors_trend_full"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 2, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 1, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + writer.set_default_identity("svc-default", "inst-default"); + + writer.write_log_batch(&[]); + writer.write_log_batch(&[ + json!({ + "body": "cpu high", + "severity_text": "Error", + "source": "kernel.log", + "watch_target": "sys/fs", + "service": {"name": "svc-x", "instance_id": "inst-x"} + }), + json!({ + "body": "cpu high", + "severity_text": "Error", + "source": "kernel.log", + "watch_target": "sys/fs", + "service": {"name": "svc-x", "instance_id": "inst-x"} + }), + json!({ + "body": "disk warning", + "severity_text": "Warn", + "service": {"name": "svc-x", "instance_id": "inst-x"} + }), + ]); + + writer.write_json_line(&json!({ + "service": {"name": "svc-y", "instance_id": "inst-y"}, + "cpu": {"usage": [1, 2], "temp": 3.5}, + "memory": {"used": 1024}, + "non_numeric": "skip", + "mixed": [{"v": 4}, true] + })); + + writer.flush(); + + let file = File::open(dir.join("metrics-trend.parquet")).expect("open trend parquet"); + let mut reader = ParquetRecordBatchReaderBuilder::try_new(file) + .expect("build parquet reader") + .build() + .expect("create record batch reader"); + + let mut metric_keys = Vec::new(); + for maybe_batch in &mut reader { + let batch = maybe_batch.expect("read trend batch"); + let key_col = batch + .column_by_name("metric_key") + .expect("metric_key column") + .as_any() + .downcast_ref::() + .expect("metric_key should be utf8"); + for idx in 0..key_col.len() { + metric_keys.push(key_col.value(idx).to_string()); + } + } + + assert!(metric_keys.iter().any(|k| k == "log.count")); + assert!(metric_keys.iter().any(|k| k == "log.severity.error.count")); + assert!(metric_keys + .iter() + .any(|k| k == "log.source.kernel_log.count")); + assert!(metric_keys + .iter() + .any(|k| k == "log.watch_target.sys_fs.count")); + assert!(metric_keys + .iter() + .any(|k| k.starts_with("log.topk.signature."))); + assert!(metric_keys.iter().any(|k| k == "cpu.usage.0")); + assert!(metric_keys.iter().any(|k| k == "cpu.usage.1")); + assert!(metric_keys.iter().any(|k| k == "cpu.temp")); + assert!(metric_keys.iter().any(|k| k == "memory.used")); + assert!(metric_keys.iter().any(|k| k == "mixed.0.v")); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn forensic_and_lossless_modes_write_expected_files() { + let dir_forensic = unique_temp_dir("host_collectors_forensic_full"); + fs::create_dir_all(&dir_forensic).expect("create forensic temp directory"); + + let forensic_config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir_forensic.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Forensic, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let mut forensic_writer = JsonArchiveWriter::from_config(&forensic_config); + forensic_writer.write_json_line(&json!({ + "body": "snapshot", + "severity_text": "Info", + "source": "sensor", + "watch_target": "cpu" + })); + forensic_writer.write_log_batch(&[json!({ + "body": "log", + "severity_text": "Warn", + "source": "syslog", + "watch_target": "disk" + })]); + + let forensic_file = dir_forensic.join("metrics-forensic.parquet"); + assert!(forensic_file.exists()); + + let dir_lossless = unique_temp_dir("host_collectors_lossless_full"); + fs::create_dir_all(&dir_lossless).expect("create lossless temp directory"); + let lossless_config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir_lossless.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Lossless, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let mut lossless_writer = JsonArchiveWriter::from_config(&lossless_config); + lossless_writer.write_json_line(&json!({"body": "snapshot"})); + lossless_writer.write_log_batch(&[json!({"body": "log"})]); + + let lossless_file = dir_lossless.join("metrics-lossless.parquet"); + assert!(lossless_file.exists()); + + fs::remove_dir_all(&dir_forensic).expect("remove forensic temp directory"); + fs::remove_dir_all(&dir_lossless).expect("remove lossless temp directory"); +} + +#[test] +fn archive_writer_trait_dispatch_and_error_path_are_covered() { + let dir = unique_temp_dir("host_collectors_trait_error"); + fs::create_dir_all(&dir).expect("create temp directory"); + let blocker = dir.join("not_a_dir"); + fs::write(&blocker, b"file blocks create_dir_all").expect("create blocker file"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: blocker.to_string_lossy().into_owned(), + max_file_bytes: 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + + let archive_writer: &mut dyn ArchiveWriter = &mut writer; + archive_writer.write_snapshot(&json!({"metric": 1})); + archive_writer.write_log_batch(&[json!({"body": "x"})]); + + assert!(!archive_writer.is_healthy()); + assert!(archive_writer.last_error().is_some()); + assert_eq!(archive_writer.total_records(), 0); + assert_eq!(archive_writer.total_bytes(), 0); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn internal_writer_branches_for_empty_inputs_and_rotation_conditions() { + let dir = unique_temp_dir("host_collectors_internal_branches"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 100, + retain_files: 0, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Forensic, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + + writer + .write_trend_window(0, HashMap::new()) + .expect("empty trend window should be ok"); + writer + .write_forensic_rows(Vec::new()) + .expect("empty forensic rows should be ok"); + + let missing_path = dir.join("missing.parquet"); + writer + .rotate_if_needed(&missing_path.to_string_lossy()) + .expect("missing file should skip rotate"); + + let small_path = dir.join("small.parquet"); + fs::write(&small_path, b"tiny").expect("create small file"); + writer + .rotate_if_needed(&small_path.to_string_lossy()) + .expect("small file should skip rotate"); + writer + .prune_rotated_files(&small_path.to_string_lossy()) + .expect("prune with retain_files=0 should be ok"); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn internal_types_defaults_and_remaining_branches_are_covered() { + let _fmt_default = ArchiveFormat::default(); + let _mode_default = ArchiveMode::default(); + let _comp_default = ArchiveCompression::default(); + + let mut stats = super::TrendStats::new(10.0); + stats.update(5.0); + stats.update(20.0); + assert_eq!(stats.min, 5.0); + assert_eq!(stats.max, 20.0); + assert_eq!(stats.count, 3); + assert!(stats.avg() > 0.0); + + let mut zero_stats = stats.clone(); + zero_stats.count = 0; + assert_eq!(zero_stats.avg(), 0.0); + + let key = super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "k".to_string(), + }; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + std::hash::Hash::hash(&key, &mut hasher); + assert_ne!(std::hash::Hasher::finish(&hasher), 0); + + let _trend_row = super::TrendRow { + window_start_unix_secs: 0, + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "k".to_string(), + min: 1.0, + max: 2.0, + avg: 1.5, + count: 2, + first: 1.0, + last: 2.0, + written_at_unix_ms: 0, + }; + + let _forensic_row = super::ForensicRow { + observed_unix_ms: 0, + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + record_type: "log".to_string(), + severity: Some("warn".to_string()), + source: Some("src".to_string()), + watch_target: Some("target".to_string()), + message_signature: Some(42), + sample_message: Some("msg".to_string()), + payload_json: "{}".to_string(), + }; + + let dir = unique_temp_dir("host_collectors_internal_remaining"); + fs::create_dir_all(&dir).expect("create temp directory"); + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1, + retain_files: 2, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 1, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + + let now = super::align_window_start(super::now_unix_secs(), 1); + let old_window = now.saturating_sub(1); + let mut old_values = HashMap::new(); + old_values.insert( + super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric.old".to_string(), + }, + super::TrendStats::new(1.0), + ); + writer.trend_windows.insert(old_window, old_values); + + writer + .flush_closed_windows(now) + .expect("closed windows should flush"); + + let trend_file = dir.join("metrics-trend.parquet"); + assert!(trend_file.exists()); + + fs::write(&trend_file, vec![b'x'; 20]).expect("seed oversized current file"); + fs::write(format!("{}.1", trend_file.to_string_lossy()), b"older-1") + .expect("seed rotated file 1"); + fs::write(format!("{}.2", trend_file.to_string_lossy()), b"older-2") + .expect("seed rotated file 2"); + fs::write(format!("{}.4", trend_file.to_string_lossy()), b"prune-me") + .expect("seed prune candidate"); + + writer + .rotate_if_needed(&trend_file.to_string_lossy()) + .expect("oversized file should rotate"); + writer + .prune_rotated_files(&trend_file.to_string_lossy()) + .expect("extra rotated files should prune"); + assert!(!PathBuf::from(format!("{}.4", trend_file.to_string_lossy())).exists()); + + let disabled = ArchiveStorageConfig::disabled("metrics"); + let mut disabled_writer = JsonArchiveWriter::from_config(&disabled); + disabled_writer.write_log_batch(&[json!({"body": "noop"})]); + assert_eq!(disabled_writer.total_records, 0); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn derived_traits_and_direct_impl_methods_are_exercised() { + let cfg = ArchiveStorageConfig { + enabled: true, + archive_dir: unique_temp_dir("host_collectors_direct_impl") + .to_string_lossy() + .into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 1, + compression: ArchiveCompression::Zstd, + }; + fs::create_dir_all(&cfg.archive_dir).expect("create temp directory"); + + let cfg_clone = cfg.clone(); + let _cfg_dbg = format!("{cfg_clone:?}"); + + let mode = ArchiveMode::Trend; + let _mode_clone = mode.clone(); + let _mode_dbg = format!("{mode:?}"); + let format_kind = ArchiveFormat::Parquet; + let _fmt_clone = format_kind.clone(); + let _fmt_dbg = format!("{format_kind:?}"); + let comp = ArchiveCompression::Zstd; + let _comp_clone = comp.clone(); + let _comp_dbg = format!("{comp:?}"); + + let mut writer = JsonArchiveWriter::from_config(&cfg); + let _writer_dbg = format!( + "enabled={} records={} bytes={}", + writer.is_healthy(), + writer.total_records(), + writer.total_bytes() + ); + + let snap = json!({"service_name": "svc", "instance_id": "inst", "cpu": {"v": 1}}); + writer + .write_snapshot_impl(&snap) + .expect("direct snapshot impl call"); + writer + .write_log_batch_impl(&[]) + .expect("direct empty log batch impl call"); + writer + .write_log_batch_impl(&[json!({"body": "x", "severity_text": "Info"})]) + .expect("direct non-empty log batch impl call"); + writer + .ingest_snapshot_to_trend(&json!([1, 2, 3])) + .expect("direct ingest snapshot for non-object payload"); + writer + .ingest_logs_to_trend(&[json!({"body": "x"})]) + .expect("direct ingest logs to trend"); + + let row = super::TrendRow { + window_start_unix_secs: 1, + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric".to_string(), + min: 1.0, + max: 1.0, + avg: 1.0, + count: 1, + first: 1.0, + last: 1.0, + written_at_unix_ms: 1, + }; + let row_clone = row.clone(); + let _row_dbg = format!("{row_clone:?}"); + + let frow = super::ForensicRow { + observed_unix_ms: 1, + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + record_type: "log".to_string(), + severity: None, + source: None, + watch_target: None, + message_signature: None, + sample_message: None, + payload_json: "{}".to_string(), + }; + let frow_clone = frow.clone(); + let _frow_dbg = format!("{frow_clone:?}"); + + let key = super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric".to_string(), + }; + let key_clone = key.clone(); + let _key_dbg = format!("{key_clone:?}"); + assert_eq!(key, key_clone); + + let filter = PrefixFilter::new(vec!["a".to_string()], vec!["b".to_string()]); + let filter_clone = filter.clone(); + let _filter_dbg = format!("{filter_clone:?}"); + + let settings = test_settings("grpc"); + let settings_clone = settings.clone(); + let _settings_dbg = format!("{settings_clone:?}"); + + fs::remove_dir_all(&cfg.archive_dir).expect("remove temp directory"); +} + +#[test] +fn add_trend_point_counts_first_sample_once() { + let dir = unique_temp_dir("host_collectors_trend_count_once"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + let window = super::align_window_start(super::now_unix_secs(), 60); + + writer.add_trend_point( + window, + super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric".to_string(), + }, + 10.0, + ); + writer.add_trend_point( + window, + super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric".to_string(), + }, + 20.0, + ); + + let stats = writer + .trend_windows + .get(&window) + .and_then(|m| { + m.get(&super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric".to_string(), + }) + }) + .expect("trend stats should exist"); + assert_eq!(stats.count, 2); + assert_eq!(stats.sum, 30.0); + assert_eq!(stats.first, 10.0); + assert_eq!(stats.last, 20.0); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn write_batch_preserves_existing_parquet_rows() { + let dir = unique_temp_dir("host_collectors_preserve_parquet_rows"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Forensic, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + + let mut writer = JsonArchiveWriter::from_config(&config); + writer.write_json_line(&json!({"body": "first", "severity_text": "Info"})); + writer.write_json_line(&json!({"body": "second", "severity_text": "Warn"})); + + let file = File::open(dir.join("metrics-forensic.parquet")).expect("open forensic parquet"); + let mut reader = ParquetRecordBatchReaderBuilder::try_new(file) + .expect("build parquet reader") + .build() + .expect("create record batch reader"); + + let mut total_rows = 0usize; + let mut payloads = Vec::new(); + for maybe_batch in &mut reader { + let batch = maybe_batch.expect("read batch"); + total_rows += batch.num_rows(); + let payload_col = batch + .column_by_name("payload_json") + .expect("payload_json column") + .as_any() + .downcast_ref::() + .expect("payload_json should be utf8"); + for i in 0..payload_col.len() { + payloads.push(payload_col.value(i).to_string()); + } + } + + assert_eq!(total_rows, 2); + assert!(payloads.iter().any(|p| p.contains("\"first\""))); + assert!(payloads.iter().any(|p| p.contains("\"second\""))); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn flush_methods_cover_window_write_lines() { + let dir = unique_temp_dir("host_collectors_flush_window_lines"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let mut writer = JsonArchiveWriter::from_config(&config); + + let mut closed_values = HashMap::new(); + closed_values.insert( + super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric.closed".to_string(), + }, + super::TrendStats::new(1.0), + ); + let mut open_values = HashMap::new(); + open_values.insert( + super::TrendKey { + service_name: "svc".to_string(), + instance_id: "inst".to_string(), + metric_key: "metric.open".to_string(), + }, + super::TrendStats::new(2.0), + ); + + writer.trend_windows.insert(100, closed_values); + writer.trend_windows.insert(200, open_values); + writer + .flush_closed_windows(150) + .expect("closed windows flush should succeed"); + + assert!(!writer.trend_windows.contains_key(&100)); + assert!(writer.trend_windows.contains_key(&200)); + + writer + .flush_all_trend_windows() + .expect("flush all windows should succeed"); + assert!(writer.trend_windows.is_empty()); + + fs::remove_dir_all(&dir).expect("remove temp directory"); +} + +#[test] +fn read_existing_batches_errors_on_schema_mismatch() { + let dir = unique_temp_dir("host_collectors_schema_mismatch"); + fs::create_dir_all(&dir).expect("create temp directory"); + + let trend_config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Trend, + window_secs: 1, + compression: ArchiveCompression::Zstd, + }; + let mut trend_writer = JsonArchiveWriter::from_config(&trend_config); + trend_writer.write_json_line(&json!({"cpu": {"value": 1.0}})); + trend_writer.flush(); + + let forensic_config = ArchiveStorageConfig { + enabled: true, + archive_dir: dir.to_string_lossy().into_owned(), + max_file_bytes: 1024 * 1024, + retain_files: 1, + file_stem: "metrics".to_string(), + format: ArchiveFormat::Parquet, + mode: ArchiveMode::Forensic, + window_secs: 60, + compression: ArchiveCompression::Zstd, + }; + let forensic_writer = JsonArchiveWriter::from_config(&forensic_config); + let mismatch = forensic_writer + .read_existing_batches( + &dir.join("metrics-trend.parquet").to_string_lossy(), + super::forensic_schema(), + ) + .expect_err("schema mismatch should return an error"); + + assert!(mismatch.to_string().contains("schema mismatch")); + fs::remove_dir_all(&dir).expect("remove temp directory"); +} diff --git a/grafana/docker.json b/grafana/docker.json index 2e98af0..7b31bb9 100644 --- a/grafana/docker.json +++ b/grafana/docker.json @@ -10,10 +10,30 @@ } ], "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } ], "editable": true, "id": null, @@ -22,29 +42,67 @@ "schemaVersion": 39, "version": 3, "refresh": "10s", - "tags": ["ojo", "docker"], - "time": { "from": "now-15m", "to": "now" }, + "tags": [ + "ojo", + "docker" + ], + "time": { + "from": "now-15m", + "to": "now" + }, "templating": { "list": [ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_docker_source_available, service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_docker_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_docker_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } }, { "name": "container", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_docker_container_cpu_ratio{service_name=~\"$service\",container_name!=\"\"}, container_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_docker_container_cpu_ratio{job=~\"$service\",instance=~\"$instance\",container_name!=\"\"}, container_name)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -53,151 +111,302 @@ "id": 1, "type": "stat", "title": "Source Available", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_docker_source_available{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 0, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_docker_source_available{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "stat", "title": "Containers Total", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_docker_containers_total{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 5, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_docker_containers_total{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 0 + } }, { "id": 3, "type": "stat", "title": "Containers Running", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_docker_containers_running{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 10, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_docker_containers_running{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 0 + } }, { "id": 4, "type": "stat", "title": "Containers Stopped", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_docker_containers_stopped{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 15, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_docker_containers_stopped{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 0 + } }, { "id": 14, "type": "stat", "title": "Labeled Containers", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "count(count by (container_name) (system_docker_container_cpu_ratio{service_name=~\"$service\",container_name!=\"\"}))" + "expr": "count(count by (container_name) (system_docker_container_cpu_ratio{job=~\"$service\",instance=~\"$instance\",container_name!=\"\"}))" } ], - "gridPos": { "h": 5, "w": 4, "x": 20, "y": 0 } + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 0 + } }, { "id": 6, "type": "timeseries", "title": "CPU Ratio by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "100 * avg by (container_name) (system_docker_container_cpu_ratio{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"})", + "expr": "100 * avg by (container_name) (system_docker_container_cpu_ratio{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"})", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 24, "x": 0, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 5 + } }, { "id": 10, "type": "timeseries", "title": "Memory Usage by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg by (container_name) (system_docker_container_memory_usage_bytes{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"})", + "expr": "avg by (container_name) (system_docker_container_memory_usage_bytes{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"})", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + } }, { "id": 11, "type": "timeseries", "title": "Memory Limit by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg by (container_name) (system_docker_container_memory_limit_bytes{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"})", + "expr": "avg by (container_name) (system_docker_container_memory_limit_bytes{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"})", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + } }, { "id": 12, "type": "timeseries", "title": "Network RX/s by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum by (container_name) (deriv(system_docker_container_network_rx_bytes{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"}[5m]))", + "expr": "sum by (container_name) (deriv(system_docker_container_network_rx_bytes{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"}[5m]))", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 19 } + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 19 + } }, { "id": 15, "type": "timeseries", "title": "Network TX/s by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum by (container_name) (deriv(system_docker_container_network_tx_bytes{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"}[5m]))", + "expr": "sum by (container_name) (deriv(system_docker_container_network_tx_bytes{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"}[5m]))", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 19 } + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 19 + } }, { "id": 13, "type": "timeseries", "title": "Block IO Read/s by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum by (container_name) (deriv(system_docker_container_block_read_bytes{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"}[5m]))", + "expr": "sum by (container_name) (deriv(system_docker_container_block_read_bytes{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"}[5m]))", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 26 } + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + } }, { "id": 16, "type": "timeseries", "title": "Block IO Write/s by Container", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum by (container_name) (deriv(system_docker_container_block_write_bytes{service_name=~\"$service\",container_name!=\"\",container_name=~\"$container\"}[5m]))", + "expr": "sum by (container_name) (deriv(system_docker_container_block_write_bytes{job=~\"$service\",instance=~\"$instance\",container_name!=\"\",container_name=~\"$container\"}[5m]))", "legendFormat": "{{container_name}}" } ], - "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 26 } + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + } } ] } diff --git a/grafana/gpu.json b/grafana/gpu.json index 9e0e4cf..fc865f7 100644 --- a/grafana/gpu.json +++ b/grafana/gpu.json @@ -10,10 +10,30 @@ } ], "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } ], "editable": true, "id": null, @@ -22,29 +42,67 @@ "schemaVersion": 39, "version": 3, "refresh": "10s", - "tags": ["ojo", "gpu"], - "time": { "from": "now-15m", "to": "now" }, + "tags": [ + "ojo", + "gpu" + ], + "time": { + "from": "now-15m", + "to": "now" + }, "templating": { "list": [ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_gpu_source_available, service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_gpu_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_gpu_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } }, { "name": "gpuName", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_gpu_utilization_ratio{service_name=~\"$service\",gpu_name!=\"\"}, gpu_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_gpu_utilization_ratio{job=~\"$service\",instance=~\"$instance\",gpu_name!=\"\"}, gpu_name)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -53,109 +111,231 @@ "id": 1, "type": "stat", "title": "Source Available", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_gpu_source_available{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 0, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_gpu_source_available{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "stat", "title": "GPU Devices", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_gpu_devices{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 5, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_gpu_devices{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 5, + "y": 0 + } }, { "id": 3, "type": "stat", "title": "Throttled", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_gpu_throttled{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 5, "x": 10, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_gpu_throttled{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 5, + "x": 10, + "y": 0 + } }, { "id": 9, "type": "stat", "title": "Avg Power Draw", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "avg(system_gpu_power_watts{service_name=~\"$service\"})" }], - "fieldConfig": { "defaults": { "unit": "watt" }, "overrides": [] }, - "gridPos": { "h": 5, "w": 5, "x": 15, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "avg(system_gpu_power_watts{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "fieldConfig": { + "defaults": { + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 15, + "y": 0 + } }, { "id": 5, "type": "timeseries", "title": "GPU Utilization by Device", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "100 * avg by (gpu_name) (system_gpu_utilization_ratio{service_name=~\"$service\",gpu_name=~\"$gpuName\"})", + "expr": "100 * avg by (gpu_name) (system_gpu_utilization_ratio{job=~\"$service\",instance=~\"$instance\",gpu_name=~\"$gpuName\"})", "legendFormat": "{{gpu_name}}" } ], - "fieldConfig": { "defaults": { "unit": "percent" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + } }, { "id": 6, "type": "timeseries", "title": "Temperature by Device", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg by (gpu_name) (system_gpu_temperature_celsius{service_name=~\"$service\",gpu_name=~\"$gpuName\"})", + "expr": "avg by (gpu_name) (system_gpu_temperature_celsius{job=~\"$service\",instance=~\"$instance\",gpu_name=~\"$gpuName\"})", "legendFormat": "{{gpu_name}}" } ], - "fieldConfig": { "defaults": { "unit": "celsius" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + } }, { "id": 7, "type": "timeseries", "title": "Memory Used by Device", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg by (gpu_name) (system_gpu_memory_used_bytes{service_name=~\"$service\",gpu_name=~\"$gpuName\"})", + "expr": "avg by (gpu_name) (system_gpu_memory_used_bytes{job=~\"$service\",instance=~\"$instance\",gpu_name=~\"$gpuName\"})", "legendFormat": "{{gpu_name}}" } ], - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + } }, { "id": 8, "type": "timeseries", "title": "Memory Total by Device", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg by (gpu_name) (system_gpu_memory_total_bytes{service_name=~\"$service\",gpu_name=~\"$gpuName\"})", + "expr": "avg by (gpu_name) (system_gpu_memory_total_bytes{job=~\"$service\",instance=~\"$instance\",gpu_name=~\"$gpuName\"})", "legendFormat": "{{gpu_name}}" } ], - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + } }, { "id": 10, "type": "timeseries", "title": "Power Draw", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_gpu_power_watts{service_name=~\"$service\"})", + "expr": "avg(system_gpu_power_watts{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Avg Power" } ], - "fieldConfig": { "defaults": { "unit": "watt" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 24, "x": 0, "y": 19 } + "fieldConfig": { + "defaults": { + "unit": "watt" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 19 + } } ] } diff --git a/grafana/mysql.json b/grafana/mysql.json index 3fc49fb..2187359 100644 --- a/grafana/mysql.json +++ b/grafana/mysql.json @@ -10,10 +10,30 @@ } ], "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } ], "editable": true, "id": null, @@ -22,19 +42,50 @@ "schemaVersion": 39, "version": 3, "refresh": "10s", - "tags": ["ojo", "mysql"], - "time": { "from": "now-15m", "to": "now" }, + "tags": [ + "ojo", + "mysql" + ], + "time": { + "from": "now-15m", + "to": "now" + }, "templating": { "list": [ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_mysql_source_available, service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_mysql_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_mysql_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -43,107 +94,206 @@ "id": 1, "type": "stat", "title": "Source Available", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_mysql_source_available{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_mysql_source_available{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "stat", "title": "MySQL Up", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_mysql_up{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_mysql_up{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + } }, { "id": 3, "type": "stat", "title": "Connections", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_mysql_connections{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_mysql_connections{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + } }, { "id": 4, "type": "stat", "title": "Threads Running", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_mysql_threads_running{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_mysql_threads_running{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + } }, { "id": 5, "type": "timeseries", "title": "Queries / Second", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_mysql_queries_rate_per_second{service_name=~\"$service\"})", + "expr": "avg(system_mysql_queries_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Queries/s" } ], - "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + } }, { "id": 6, "type": "timeseries", "title": "Network Bytes / Second", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_mysql_bytes_received_rate_per_second{service_name=~\"$service\"})", + "expr": "sum(system_mysql_bytes_received_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Received" }, { "refId": "B", - "expr": "sum(system_mysql_bytes_sent_rate_per_second{service_name=~\"$service\"})", + "expr": "sum(system_mysql_bytes_sent_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Sent" } ], - "fieldConfig": { "defaults": { "unit": "Bps" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + } }, { "id": 7, "type": "timeseries", "title": "Query Counters (Total)", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_mysql_queries_total{service_name=~\"$service\"})", + "expr": "sum(system_mysql_queries_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Queries" }, { "refId": "B", - "expr": "sum(system_mysql_slow_queries_total{service_name=~\"$service\"})", + "expr": "sum(system_mysql_slow_queries_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Slow Queries" } ], - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 } + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + } }, { "id": 8, "type": "timeseries", "title": "Network Bytes (Total)", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_mysql_bytes_received_total{service_name=~\"$service\"})", + "expr": "sum(system_mysql_bytes_received_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Received" }, { "refId": "B", - "expr": "sum(system_mysql_bytes_sent_total{service_name=~\"$service\"})", + "expr": "sum(system_mysql_bytes_sent_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Sent" } ], - "fieldConfig": { "defaults": { "unit": "bytes" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + } } ] } diff --git a/grafana/nfs-client.json b/grafana/nfs-client.json index d99d6ab..992687f 100644 --- a/grafana/nfs-client.json +++ b/grafana/nfs-client.json @@ -10,10 +10,30 @@ } ], "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } ], "editable": true, "id": null, @@ -22,19 +42,50 @@ "schemaVersion": 39, "version": 2, "refresh": "10s", - "tags": ["ojo", "nfs"], - "time": { "from": "now-15m", "to": "now" }, + "tags": [ + "ojo", + "nfs" + ], + "time": { + "from": "now-15m", + "to": "now" + }, "templating": { "list": [ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_nfs_client_source_available, service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_nfs_client_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_nfs_client_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -43,71 +94,131 @@ "id": 1, "type": "stat", "title": "Source Available", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_nfs_client_source_available{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_nfs_client_source_available{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "stat", "title": "NFS Mounts", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_nfs_client_mounts{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_nfs_client_mounts{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + } }, { "id": 3, "type": "timeseries", "title": "RPC Calls / Second", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_nfs_client_rpc_calls_rate_per_second{service_name=~\"$service\"})", + "expr": "avg(system_nfs_client_rpc_calls_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "RPC Calls/s" } ], - "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + } }, { "id": 5, "type": "timeseries", "title": "RPC Retransmissions / Second", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_nfs_client_rpc_retransmissions_rate_per_second{service_name=~\"$service\"})", + "expr": "avg(system_nfs_client_rpc_retransmissions_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Retransmissions/s" } ], - "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + } }, { "id": 4, "type": "timeseries", "title": "RPC Counters (Total)", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_nfs_client_rpc_calls_total{service_name=~\"$service\"})", + "expr": "sum(system_nfs_client_rpc_calls_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Calls" }, { "refId": "B", - "expr": "sum(system_nfs_client_rpc_retransmissions_total{service_name=~\"$service\"})", + "expr": "sum(system_nfs_client_rpc_retransmissions_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Retransmissions" }, { "refId": "C", - "expr": "sum(system_nfs_client_rpc_auth_refreshes_total{service_name=~\"$service\"})", + "expr": "sum(system_nfs_client_rpc_auth_refreshes_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Auth Refreshes" } ], - "gridPos": { "h": 7, "w": 24, "x": 0, "y": 12 } + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 12 + } } ] } diff --git a/grafana/nginx.json b/grafana/nginx.json index 18a8fdd..aafac0f 100644 --- a/grafana/nginx.json +++ b/grafana/nginx.json @@ -59,7 +59,25 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_nginx_source_available, service_name)", + "query": "label_values(system_nginx_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_nginx_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, @@ -79,7 +97,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_nginx_up{service_name=~\"$service\"})" + "expr": "max(system_nginx_up{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -100,7 +118,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_nginx_source_available{service_name=~\"$service\"})" + "expr": "max(system_nginx_source_available{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -121,7 +139,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_nginx_connections_active{service_name=~\"$service\"})" + "expr": "max(system_nginx_connections_active{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -142,7 +160,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_nginx_requests_rate_per_second{service_name=~\"$service\"} or rate(system_nginx_requests_total{service_name=~\"$service\"}[1m]))" + "expr": "max(system_nginx_requests_rate_per_second{job=~\"$service\",instance=~\"$instance\"} or rate(system_nginx_requests_total{job=~\"$service\",instance=~\"$instance\"}[1m]))" } ], "fieldConfig": { @@ -168,19 +186,19 @@ "targets": [ { "refId": "A", - "expr": "system_nginx_connections_active{service_name=~\"$service\"}" + "expr": "system_nginx_connections_active{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_nginx_connections_reading{service_name=~\"$service\"}" + "expr": "system_nginx_connections_reading{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_nginx_connections_writing{service_name=~\"$service\"}" + "expr": "system_nginx_connections_writing{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_nginx_connections_waiting{service_name=~\"$service\"}" + "expr": "system_nginx_connections_waiting{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -201,11 +219,11 @@ "targets": [ { "refId": "A", - "expr": "system_nginx_requests_rate_per_second{service_name=~\"$service\"} or rate(system_nginx_requests_total{service_name=~\"$service\"}[1m])" + "expr": "system_nginx_requests_rate_per_second{job=~\"$service\",instance=~\"$instance\"} or rate(system_nginx_requests_total{job=~\"$service\",instance=~\"$instance\"}[1m])" }, { "refId": "B", - "expr": "system_nginx_connections_accepted_rate_per_second{service_name=~\"$service\"} or rate(system_nginx_connections_accepted_total{service_name=~\"$service\"}[1m])" + "expr": "system_nginx_connections_accepted_rate_per_second{job=~\"$service\",instance=~\"$instance\"} or rate(system_nginx_connections_accepted_total{job=~\"$service\",instance=~\"$instance\"}[1m])" } ], "gridPos": { @@ -226,15 +244,15 @@ "targets": [ { "refId": "A", - "expr": "system_nginx_connections_accepted_total{service_name=~\"$service\"}" + "expr": "system_nginx_connections_accepted_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_nginx_connections_handled_total{service_name=~\"$service\"}" + "expr": "system_nginx_connections_handled_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_nginx_requests_total{service_name=~\"$service\"}" + "expr": "system_nginx_requests_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { diff --git a/grafana/postgres.json b/grafana/postgres.json index 1888b32..01877da 100644 --- a/grafana/postgres.json +++ b/grafana/postgres.json @@ -10,10 +10,30 @@ } ], "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } ], "editable": true, "id": null, @@ -22,19 +42,50 @@ "schemaVersion": 39, "version": 3, "refresh": "10s", - "tags": ["ojo", "postgres"], - "time": { "from": "now-15m", "to": "now" }, + "tags": [ + "ojo", + "postgres" + ], + "time": { + "from": "now-15m", + "to": "now" + }, "templating": { "list": [ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_postgres_source_available, service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_postgres_source_available, job)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_postgres_source_available{job=~\"$service\"}, instance)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -43,105 +94,194 @@ "id": 1, "type": "stat", "title": "Source Available", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_postgres_source_available{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_postgres_source_available{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "stat", "title": "Postgres Up", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_postgres_up{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_postgres_up{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + } }, { "id": 3, "type": "stat", "title": "Connections", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_postgres_connections{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_postgres_connections{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + } }, { "id": 4, "type": "stat", "title": "Deadlocks", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_postgres_deadlocks_total{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_postgres_deadlocks_total{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + } }, { "id": 5, "type": "timeseries", "title": "Transactions / Second", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_postgres_transactions_committed_rate_per_second{service_name=~\"$service\"})", + "expr": "avg(system_postgres_transactions_committed_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Committed" }, { "refId": "B", - "expr": "avg(system_postgres_transactions_rolled_back_rate_per_second{service_name=~\"$service\"})", + "expr": "avg(system_postgres_transactions_rolled_back_rate_per_second{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Rolled Back" } ], - "fieldConfig": { "defaults": { "unit": "ops" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 5 + } }, { "id": 6, "type": "timeseries", "title": "Buffer Blocks Read vs Hit", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_postgres_blocks_read_total{service_name=~\"$service\"})", + "expr": "sum(system_postgres_blocks_read_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Blocks Read" }, { "refId": "B", - "expr": "sum(system_postgres_blocks_hit_total{service_name=~\"$service\"})", + "expr": "sum(system_postgres_blocks_hit_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Blocks Hit (cache)" } ], - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 } + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 5 + } }, { "id": 7, "type": "timeseries", "title": "Transaction Counters (Total)", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_postgres_transactions_committed_total{service_name=~\"$service\"})", + "expr": "sum(system_postgres_transactions_committed_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Committed" }, { "refId": "B", - "expr": "sum(system_postgres_transactions_rolled_back_total{service_name=~\"$service\"})", + "expr": "sum(system_postgres_transactions_rolled_back_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Rolled Back" } ], - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 } + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + } }, { "id": 8, "type": "timeseries", "title": "Deadlocks (Total)", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "sum(system_postgres_deadlocks_total{service_name=~\"$service\"})", + "expr": "sum(system_postgres_deadlocks_total{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Deadlocks" } ], - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 } + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + } } ] } diff --git a/grafana/redis.json b/grafana/redis.json index 79b9106..9a92c41 100644 --- a/grafana/redis.json +++ b/grafana/redis.json @@ -59,7 +59,25 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_redis_source_available, service_name)", + "query": "label_values(system_redis_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_redis_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, @@ -79,7 +97,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_redis_up{service_name=~\"$service\"})" + "expr": "max(system_redis_up{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -100,7 +118,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_redis_source_available{service_name=~\"$service\"})" + "expr": "max(system_redis_source_available{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -121,7 +139,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_redis_clients_connected{service_name=~\"$service\"})" + "expr": "max(system_redis_clients_connected{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -142,7 +160,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_redis_clients_blocked{service_name=~\"$service\"})" + "expr": "max(system_redis_clients_blocked{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -163,7 +181,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_redis_commands_processed_rate_per_second{service_name=~\"$service\"} or rate(system_redis_commands_processed_total{service_name=~\"$service\"}[1m]))" + "expr": "max(system_redis_commands_processed_rate_per_second{job=~\"$service\",instance=~\"$instance\"} or rate(system_redis_commands_processed_total{job=~\"$service\",instance=~\"$instance\"}[1m]))" } ], "fieldConfig": { @@ -189,7 +207,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_redis_keyspace_hit_ratio{service_name=~\"$service\"})" + "expr": "max(system_redis_keyspace_hit_ratio{job=~\"$service\",instance=~\"$instance\"})" } ], "fieldConfig": { @@ -217,11 +235,11 @@ "targets": [ { "refId": "A", - "expr": "system_redis_memory_used_bytes{service_name=~\"$service\"}" + "expr": "system_redis_memory_used_bytes{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_redis_memory_max_bytes{service_name=~\"$service\"}" + "expr": "system_redis_memory_max_bytes{job=~\"$service\",instance=~\"$instance\"}" } ], "fieldConfig": { @@ -247,11 +265,11 @@ "targets": [ { "refId": "A", - "expr": "system_redis_commands_processed_rate_per_second{service_name=~\"$service\"} or rate(system_redis_commands_processed_total{service_name=~\"$service\"}[1m])" + "expr": "system_redis_commands_processed_rate_per_second{job=~\"$service\",instance=~\"$instance\"} or rate(system_redis_commands_processed_total{job=~\"$service\",instance=~\"$instance\"}[1m])" }, { "refId": "B", - "expr": "system_redis_connections_received_rate_per_second{service_name=~\"$service\"}" + "expr": "system_redis_connections_received_rate_per_second{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -272,15 +290,15 @@ "targets": [ { "refId": "A", - "expr": "system_redis_commands_processed_total{service_name=~\"$service\"}" + "expr": "system_redis_commands_processed_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_redis_connections_received_total{service_name=~\"$service\"}" + "expr": "system_redis_connections_received_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_redis_uptime_seconds{service_name=~\"$service\"}" + "expr": "system_redis_uptime_seconds{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -301,19 +319,19 @@ "targets": [ { "refId": "A", - "expr": "system_redis_keyspace_hits_total{service_name=~\"$service\"}" + "expr": "system_redis_keyspace_hits_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_redis_keyspace_misses_total{service_name=~\"$service\"}" + "expr": "system_redis_keyspace_misses_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_redis_keys_expired_total{service_name=~\"$service\"}" + "expr": "system_redis_keys_expired_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_redis_keys_evicted_total{service_name=~\"$service\"}" + "expr": "system_redis_keys_evicted_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { diff --git a/grafana/sensors.json b/grafana/sensors.json index 417d0c0..5e0672d 100644 --- a/grafana/sensors.json +++ b/grafana/sensors.json @@ -10,10 +10,30 @@ } ], "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "11.0.0" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } ], "editable": true, "id": null, @@ -22,29 +42,67 @@ "schemaVersion": 39, "version": 3, "refresh": "10s", - "tags": ["ojo", "sensors"], - "time": { "from": "now-15m", "to": "now" }, + "tags": [ + "ojo", + "sensors" + ], + "time": { + "from": "now-15m", + "to": "now" + }, "templating": { "list": [ { "name": "service", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_sensor_source_available, service_name)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_sensor_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_sensor_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } }, { "name": "sensorLabel", "type": "query", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_sensor_temperature_celsius{service_name=~\"$service\",sensor_label!=\"\"}, sensor_label)", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_sensor_temperature_celsius{job=~\"$service\",instance=~\"$instance\",sensor_label!=\"\"}, sensor_label)", "includeAll": true, "multi": true, "refresh": 1, - "current": { "selected": true, "text": "All", "value": "$__all" } + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } } ] }, @@ -53,80 +111,181 @@ "id": 1, "type": "stat", "title": "Source Available", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_sensor_source_available{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 0, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_sensor_source_available{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + } }, { "id": 2, "type": "stat", "title": "Sensors Count", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_sensor_count{service_name=~\"$service\"})" }], - "gridPos": { "h": 5, "w": 6, "x": 6, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_sensor_count{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + } }, { "id": 7, "type": "stat", "title": "Avg Temperature", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "avg(system_sensor_temperature_celsius{service_name=~\"$service\"})" }], - "fieldConfig": { "defaults": { "unit": "celsius" }, "overrides": [] }, - "gridPos": { "h": 5, "w": 6, "x": 12, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "avg(system_sensor_temperature_celsius{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + } }, { "id": 8, "type": "stat", "title": "Max Temperature", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "targets": [{ "refId": "A", "expr": "max(system_sensor_temperature_max_celsius{service_name=~\"$service\"})" }], - "fieldConfig": { "defaults": { "unit": "celsius" }, "overrides": [] }, - "gridPos": { "h": 5, "w": 6, "x": 18, "y": 0 } + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "targets": [ + { + "refId": "A", + "expr": "max(system_sensor_temperature_max_celsius{job=~\"$service\",instance=~\"$instance\"})" + } + ], + "fieldConfig": { + "defaults": { + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + } }, { "id": 4, "type": "timeseries", "title": "Temperature by Sensor", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg by (sensor_label) (system_sensor_temperature_celsius{service_name=~\"$service\",sensor_label!=\"\",sensor_label=~\"$sensorLabel\"}) or label_replace(avg(system_sensor_temperature_celsius{service_name=~\"$service\",sensor_label=\"\"}), \"sensor_label\", \"aggregate\", \"__name__\", \".*\")", + "expr": "avg by (sensor_label) (system_sensor_temperature_celsius{job=~\"$service\",instance=~\"$instance\",sensor_label!=\"\",sensor_label=~\"$sensorLabel\"}) or label_replace(avg(system_sensor_temperature_celsius{job=~\"$service\",instance=~\"$instance\",sensor_label=\"\"}), \"sensor_label\", \"aggregate\", \"__name__\", \".*\")", "legendFormat": "{{sensor_label}}" } ], - "fieldConfig": { "defaults": { "unit": "celsius" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 24, "x": 0, "y": 5 } + "fieldConfig": { + "defaults": { + "unit": "celsius" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 5 + } }, { "id": 5, "type": "timeseries", "title": "Fan RPM", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_sensor_fan_rpm{service_name=~\"$service\"})", + "expr": "avg(system_sensor_fan_rpm{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Avg Fan RPM" } ], - "fieldConfig": { "defaults": { "unit": "rotrpm" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "rotrpm" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 12 + } }, { "id": 6, "type": "timeseries", "title": "Voltage", - "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "targets": [ { "refId": "A", - "expr": "avg(system_sensor_voltage_volts{service_name=~\"$service\"})", + "expr": "avg(system_sensor_voltage_volts{job=~\"$service\",instance=~\"$instance\"})", "legendFormat": "Avg Voltage" } ], - "fieldConfig": { "defaults": { "unit": "volt" }, "overrides": [] }, - "gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 } + "fieldConfig": { + "defaults": { + "unit": "volt" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 12 + } } ] } diff --git a/grafana/syslog.json b/grafana/syslog.json index a7eba01..d5bb3ea 100644 --- a/grafana/syslog.json +++ b/grafana/syslog.json @@ -59,7 +59,25 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_syslog_source_available, service_name)", + "query": "label_values(system_syslog_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_syslog_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, @@ -79,7 +97,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_syslog_up{service_name=~\"$service\"})" + "expr": "max(system_syslog_up{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -100,7 +118,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_syslog_exporter_available{service_name=~\"$service\"})" + "expr": "max(system_syslog_exporter_available{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -121,7 +139,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_syslog_buffer_queued_records{service_name=~\"$service\"})" + "expr": "max(system_syslog_buffer_queued_records{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -142,7 +160,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_syslog_file_watch_targets_active{service_name=~\"$service\"})" + "expr": "max(system_syslog_file_watch_targets_active{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -163,19 +181,19 @@ "targets": [ { "refId": "A", - "expr": "system_syslog_source_available{service_name=~\"$service\"}" + "expr": "system_syslog_source_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_syslog_journald_available{service_name=~\"$service\"}" + "expr": "system_syslog_journald_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_syslog_etw_available{service_name=~\"$service\"}" + "expr": "system_syslog_etw_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_syslog_kernel_dmesg_available{service_name=~\"$service\"}" + "expr": "system_syslog_kernel_dmesg_available{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -196,15 +214,15 @@ "targets": [ { "refId": "A", - "expr": "rate(system_syslog_logs_collected_total{service_name=~\"$service\"}[5m])" + "expr": "rate(system_syslog_logs_collected_total{job=~\"$service\",instance=~\"$instance\"}[5m])" }, { "refId": "B", - "expr": "rate(system_syslog_logs_exported_total{service_name=~\"$service\"}[5m])" + "expr": "rate(system_syslog_logs_exported_total{job=~\"$service\",instance=~\"$instance\"}[5m])" }, { "refId": "C", - "expr": "rate(system_syslog_buffer_dropped_total{service_name=~\"$service\"}[5m])" + "expr": "rate(system_syslog_buffer_dropped_total{job=~\"$service\",instance=~\"$instance\"}[5m])" } ], "gridPos": { @@ -225,15 +243,15 @@ "targets": [ { "refId": "A", - "expr": "rate(system_syslog_logs_retry_total{service_name=~\"$service\"}[5m])" + "expr": "rate(system_syslog_logs_retry_total{job=~\"$service\",instance=~\"$instance\"}[5m])" }, { "refId": "B", - "expr": "rate(system_syslog_logs_export_errors_total{service_name=~\"$service\"}[5m])" + "expr": "rate(system_syslog_logs_export_errors_total{job=~\"$service\",instance=~\"$instance\"}[5m])" }, { "refId": "C", - "expr": "system_syslog_exporter_reconnecting{service_name=~\"$service\"}" + "expr": "system_syslog_exporter_reconnecting{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -254,11 +272,11 @@ "targets": [ { "refId": "A", - "expr": "system_syslog_logs_batch_size{service_name=~\"$service\"}" + "expr": "system_syslog_logs_batch_size{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_syslog_logs_payload_bytes{service_name=~\"$service\"}" + "expr": "system_syslog_logs_payload_bytes{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -279,15 +297,15 @@ "targets": [ { "refId": "A", - "expr": "system_syslog_file_watch_targets_configured{service_name=~\"$service\"}" + "expr": "system_syslog_file_watch_targets_configured{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_syslog_buffer_capacity_records{service_name=~\"$service\"}" + "expr": "system_syslog_buffer_capacity_records{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_syslog_buffer_queued_records{service_name=~\"$service\"}" + "expr": "system_syslog_buffer_queued_records{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -308,15 +326,15 @@ "targets": [ { "refId": "A", - "expr": "system_syslog_process_logs_available{service_name=~\"$service\"}" + "expr": "system_syslog_process_logs_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_syslog_application_logs_available{service_name=~\"$service\"}" + "expr": "system_syslog_application_logs_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_syslog_collection_errors{service_name=~\"$service\"}" + "expr": "system_syslog_collection_errors{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { diff --git a/grafana/systemd.json b/grafana/systemd.json index 101e105..9b7252a 100644 --- a/grafana/systemd.json +++ b/grafana/systemd.json @@ -59,7 +59,25 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_systemd_source_available, service_name)", + "query": "label_values(system_systemd_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_systemd_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, @@ -79,7 +97,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systemd_up{service_name=~\"$service\"})" + "expr": "max(system_systemd_up{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -100,7 +118,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systemd_source_available{service_name=~\"$service\"})" + "expr": "max(system_systemd_source_available{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -121,7 +139,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systemd_units_total{service_name=~\"$service\"})" + "expr": "max(system_systemd_units_total{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -142,7 +160,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systemd_failed_units_reported{service_name=~\"$service\"})" + "expr": "max(system_systemd_failed_units_reported{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -163,35 +181,35 @@ "targets": [ { "refId": "A", - "expr": "system_systemd_units_active{service_name=~\"$service\"}" + "expr": "system_systemd_units_active{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systemd_units_inactive{service_name=~\"$service\"}" + "expr": "system_systemd_units_inactive{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systemd_units_failed{service_name=~\"$service\"}" + "expr": "system_systemd_units_failed{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_systemd_units_activating{service_name=~\"$service\"}" + "expr": "system_systemd_units_activating{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "E", - "expr": "system_systemd_units_deactivating{service_name=~\"$service\"}" + "expr": "system_systemd_units_deactivating{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "F", - "expr": "system_systemd_units_reloading{service_name=~\"$service\"}" + "expr": "system_systemd_units_reloading{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "G", - "expr": "system_systemd_units_maintenance{service_name=~\"$service\"}" + "expr": "system_systemd_units_maintenance{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "H", - "expr": "system_systemd_units_not_found{service_name=~\"$service\"}" + "expr": "system_systemd_units_not_found{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -212,11 +230,11 @@ "targets": [ { "refId": "A", - "expr": "system_systemd_jobs_queued{service_name=~\"$service\"}" + "expr": "system_systemd_jobs_queued{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systemd_jobs_running{service_name=~\"$service\"}" + "expr": "system_systemd_jobs_running{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -237,11 +255,11 @@ "targets": [ { "refId": "A", - "expr": "system_systemd_units_active_ratio{service_name=~\"$service\"}" + "expr": "system_systemd_units_active_ratio{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systemd_units_failed_ratio{service_name=~\"$service\"}" + "expr": "system_systemd_units_failed_ratio{job=~\"$service\",instance=~\"$instance\"}" } ], "fieldConfig": { @@ -269,11 +287,11 @@ "targets": [ { "refId": "A", - "expr": "system_systemd_units_failed{service_name=~\"$service\"}" + "expr": "system_systemd_units_failed{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systemd_failed_units_reported{service_name=~\"$service\"}" + "expr": "system_systemd_failed_units_reported{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { diff --git a/grafana/systrace.json b/grafana/systrace.json index 5d1874c..ecda5de 100644 --- a/grafana/systrace.json +++ b/grafana/systrace.json @@ -59,7 +59,25 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "query": "label_values(system_systrace_source_available, service_name)", + "query": "label_values(system_systrace_source_available, job)", + "includeAll": true, + "multi": true, + "refresh": 1, + "current": { + "selected": true, + "text": "All", + "value": "$__all" + } + }, + { + "name": "instance", + "label": "Instance", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": "label_values(system_systrace_source_available{job=~\"$service\"}, instance)", "includeAll": true, "multi": true, "refresh": 1, @@ -79,7 +97,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systrace_up{service_name=~\"$service\"})" + "expr": "max(system_systrace_up{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -100,7 +118,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systrace_tracing_on{service_name=~\"$service\"})" + "expr": "max(system_systrace_tracing_on{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -121,7 +139,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systrace_processes_total{service_name=~\"$service\"})" + "expr": "max(system_systrace_processes_total{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -142,7 +160,7 @@ "targets": [ { "refId": "A", - "expr": "max(system_systrace_threads_total{service_name=~\"$service\"})" + "expr": "max(system_systrace_threads_total{job=~\"$service\",instance=~\"$instance\"})" } ], "gridPos": { @@ -160,18 +178,19 @@ "id": 5, "type": "timeseries", "title": "Scheduler Pressure", + "description": "Collector-derived per-second gauges; PromQL rate() is not needed here.", "targets": [ { "refId": "A", - "expr": "system_systrace_run_queue_depth{service_name=~\"$service\"}" + "expr": "system_systrace_run_queue_depth{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_context_switches_per_sec{service_name=~\"$service\"}" + "expr": "system_systrace_context_switches_per_sec{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_process_forks_per_sec{service_name=~\"$service\"}" + "expr": "system_systrace_process_forks_per_sec{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -189,18 +208,19 @@ "id": 6, "type": "timeseries", "title": "Kernel Event Rates", + "description": "Collector-derived per-second gauges; PromQL rate() is not needed here.", "targets": [ { "refId": "A", - "expr": "system_systrace_interrupts_per_sec{service_name=~\"$service\"}" + "expr": "system_systrace_interrupts_per_sec{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_system_calls_per_sec{service_name=~\"$service\"}" + "expr": "system_systrace_system_calls_per_sec{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_dpcs_per_sec{service_name=~\"$service\"}" + "expr": "system_systrace_dpcs_per_sec{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -221,31 +241,31 @@ "targets": [ { "refId": "S", - "expr": "system_systrace_source_available{service_name=~\"$service\"}" + "expr": "system_systrace_source_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "A", - "expr": "system_systrace_tracefs_available{service_name=~\"$service\"}" + "expr": "system_systrace_tracefs_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_etw_available{service_name=~\"$service\"}" + "expr": "system_systrace_etw_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_tracers_available{service_name=~\"$service\"}" + "expr": "system_systrace_tracers_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_systrace_etw_sessions_total{service_name=~\"$service\"}" + "expr": "system_systrace_etw_sessions_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "E", - "expr": "system_systrace_etw_sessions_running{service_name=~\"$service\"}" + "expr": "system_systrace_etw_sessions_running{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "F", - "expr": "system_systrace_etw_providers_total{service_name=~\"$service\"}" + "expr": "system_systrace_etw_providers_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -266,19 +286,19 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_events_total{service_name=~\"$service\"}" + "expr": "system_systrace_events_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_events_enabled{service_name=~\"$service\"}" + "expr": "system_systrace_events_enabled{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "100 * system_systrace_events_enabled{service_name=~\"$service\"} / clamp_min(system_systrace_events_total{service_name=~\"$service\"}, 1)" + "expr": "100 * system_systrace_events_enabled{job=~\"$service\",instance=~\"$instance\"} / clamp_min(system_systrace_events_total{job=~\"$service\",instance=~\"$instance\"}, 1)" }, { "refId": "D", - "expr": "system_systrace_event_categories_total{service_name=~\"$service\"}" + "expr": "system_systrace_event_categories_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -299,19 +319,19 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_collection_errors{service_name=~\"$service\"}" + "expr": "system_systrace_collection_errors{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_exporter_errors_total{service_name=~\"$service\"}" + "expr": "system_systrace_exporter_errors_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_exporter_available{service_name=~\"$service\"}" + "expr": "system_systrace_exporter_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_systrace_exporter_reconnecting{service_name=~\"$service\"}" + "expr": "system_systrace_exporter_reconnecting{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -332,11 +352,11 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_system_calls_source{service_name=~\"$service\"}" + "expr": "system_systrace_system_calls_source{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "100 * system_systrace_system_calls_coverage_ratio{service_name=~\"$service\"}" + "expr": "100 * system_systrace_system_calls_coverage_ratio{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -357,15 +377,15 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_trace_kernel_stack_samples_total{service_name=~\"$service\"}" + "expr": "system_systrace_trace_kernel_stack_samples_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_trace_user_stack_samples_total{service_name=~\"$service\"}" + "expr": "system_systrace_trace_user_stack_samples_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_trace_sample_lines_total{service_name=~\"$service\"}" + "expr": "system_systrace_trace_sample_lines_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -386,7 +406,7 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_buffer_total_kb{service_name=~\"$service\"}" + "expr": "system_systrace_buffer_total_kb{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -407,15 +427,15 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_events_enabled_inventory_total{service_name=~\"$service\"}" + "expr": "system_systrace_events_enabled_inventory_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_syscalls_enter_enabled{service_name=~\"$service\"}" + "expr": "system_systrace_syscalls_enter_enabled{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_syscalls_exit_enabled{service_name=~\"$service\"}" + "expr": "system_systrace_syscalls_exit_enabled{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -436,15 +456,15 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_coverage_high_value_categories_targeted{service_name=~\"$service\"}" + "expr": "system_systrace_coverage_high_value_categories_targeted{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_coverage_high_value_categories_enabled{service_name=~\"$service\"}" + "expr": "system_systrace_coverage_high_value_categories_enabled{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "100 * system_systrace_coverage_high_value_categories_enabled{service_name=~\"$service\"} / clamp_min(system_systrace_coverage_high_value_categories_targeted{service_name=~\"$service\"}, 1)" + "expr": "100 * system_systrace_coverage_high_value_categories_enabled{job=~\"$service\",instance=~\"$instance\"} / clamp_min(system_systrace_coverage_high_value_categories_targeted{job=~\"$service\",instance=~\"$instance\"}, 1)" } ], "gridPos": { @@ -465,11 +485,11 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_trace_stream_lines_captured_total{service_name=~\"$service\"}" + "expr": "system_systrace_trace_stream_lines_captured_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_trace_stream_continuity{service_name=~\"$service\"}" + "expr": "system_systrace_trace_stream_continuity{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -490,11 +510,11 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_trace_dropped_events_total{service_name=~\"$service\"}" + "expr": "system_systrace_trace_dropped_events_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_trace_overrun_events_total{service_name=~\"$service\"}" + "expr": "system_systrace_trace_overrun_events_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -515,23 +535,23 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_privileged_mode{service_name=~\"$service\"}" + "expr": "system_systrace_privileged_mode{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_ebpf_available{service_name=~\"$service\"}" + "expr": "system_systrace_ebpf_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_uprobes_available{service_name=~\"$service\"}" + "expr": "system_systrace_uprobes_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "D", - "expr": "system_systrace_usdt_available{service_name=~\"$service\"}" + "expr": "system_systrace_usdt_available{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "E", - "expr": "system_systrace_symbolizer_available{service_name=~\"$service\"}" + "expr": "system_systrace_symbolizer_available{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -552,15 +572,15 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_archive_writer_healthy{service_name=~\"$service\"}" + "expr": "system_systrace_archive_writer_healthy{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "B", - "expr": "system_systrace_archive_events_total{service_name=~\"$service\"}" + "expr": "system_systrace_archive_events_total{job=~\"$service\",instance=~\"$instance\"}" }, { "refId": "C", - "expr": "system_systrace_archive_bytes_total{service_name=~\"$service\"}" + "expr": "system_systrace_archive_bytes_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { @@ -581,7 +601,7 @@ "targets": [ { "refId": "A", - "expr": "system_systrace_runtime_probes_configured_total{service_name=~\"$service\"}" + "expr": "system_systrace_runtime_probes_configured_total{job=~\"$service\",instance=~\"$instance\"}" } ], "gridPos": { diff --git a/linux.yaml b/linux.yaml index 3a758a4..b8834e1 100644 --- a/linux.yaml +++ b/linux.yaml @@ -26,11 +26,15 @@ export: timeout_secs: 10 storage: - archive_enabled: false + archive_enabled: true archive_dir: data/ojo archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: ojo-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd # Rules: # - include/exclude are prefix-based diff --git a/services/docker/README.md b/services/docker/README.md index 6af5d57..4d6cb5d 100644 --- a/services/docker/README.md +++ b/services/docker/README.md @@ -25,6 +25,27 @@ Archive settings: - `storage.archive_max_file_bytes` - `storage.archive_retain_files` - `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries (`min/max/avg/count/first/last`) per window. +- `lossless`: full-fidelity row archival with efficient parquet + zstd compression. +- `forensic`: compatibility row mode. + +Typical files: +- `-trend.parquet` +- `-lossless.parquet` + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/docker/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` ## Run ```bash @@ -32,3 +53,10 @@ cargo run -p ojo-docker -- --config services/docker/docker.yaml ``` Use `--once` for one-shot collection. + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/docker/docker.yaml b/services/docker/docker.yaml index 195ad61..e557b73 100644 --- a/services/docker/docker.yaml +++ b/services/docker/docker.yaml @@ -29,3 +29,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: docker-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/docker/src/main.rs b/services/docker/src/main.rs index f8d53d6..85f868b 100644 --- a/services/docker/src/main.rs +++ b/services/docker/src/main.rs @@ -1,7 +1,8 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, METRIC_PREFIX_SYSTEM, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, + METRIC_PREFIX_SYSTEM, }; use opentelemetry::metrics::{Gauge, Meter}; use opentelemetry::KeyValue; @@ -101,6 +102,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[derive(Clone, Debug, Default, Serialize)] @@ -308,6 +313,7 @@ fn run() -> Result<()> { let running = Arc::new(AtomicBool::new(true)); install_signal_handler(&running); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); let mut export_state = ExportState::Pending; let mut continue_running = true; @@ -661,6 +667,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "docker-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) diff --git a/services/docker/src/tests/main_tests.rs b/services/docker/src/tests/main_tests.rs index aa2d37e..cd6579d 100644 --- a/services/docker/src/tests/main_tests.rs +++ b/services/docker/src/tests/main_tests.rs @@ -393,6 +393,10 @@ fn record_snapshot_covers_available_unavailable_and_labeled_paths() { max_file_bytes: 0, retain_files: 0, file_stem: "docker-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; @@ -459,6 +463,10 @@ fn record_snapshot_returns_early_when_labels_disabled() { max_file_bytes: 0, retain_files: 0, file_stem: "docker-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; diff --git a/services/gpu/README.md b/services/gpu/README.md index c44a9a6..0f33999 100644 --- a/services/gpu/README.md +++ b/services/gpu/README.md @@ -19,10 +19,43 @@ Important keys: - `metrics.include` / `metrics.exclude` - `storage.archive_*` -## Archive output -When enabled, each polling snapshot is persisted as NDJSON with rotation and retention. +## Storage and archive +GPU archives are parquet-based: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for long-term trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Typical files: +- `-trend.parquet` +- `-lossless.parquet` + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/gpu/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` ## Run ```bash cargo run -p ojo-gpu -- --config services/gpu/gpu.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/gpu/gpu.yaml b/services/gpu/gpu.yaml index 72c78c1..918b826 100644 --- a/services/gpu/gpu.yaml +++ b/services/gpu/gpu.yaml @@ -28,3 +28,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: gpu-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/gpu/src/main.rs b/services/gpu/src/main.rs index 53142bd..087a41a 100644 --- a/services/gpu/src/main.rs +++ b/services/gpu/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -101,6 +101,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[derive(Clone, Debug, Default, Serialize)] @@ -290,6 +294,7 @@ fn run() -> Result<()> { let running = Arc::new(AtomicBool::new(true)); install_signal_handler(&running); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); let mut export_state = ExportState::Pending; let mut continue_running = true; @@ -538,6 +543,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "gpu-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) diff --git a/services/gpu/src/platform/common.rs b/services/gpu/src/platform/common.rs index cc3d61e..30c3e8e 100644 --- a/services/gpu/src/platform/common.rs +++ b/services/gpu/src/platform/common.rs @@ -78,6 +78,7 @@ fn run_with_timeout_using_waiter( where W: FnMut(&mut Child) -> std::io::Result>, { + let program = cmd.get_program().to_string_lossy().into_owned(); cmd.stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -85,7 +86,7 @@ where let mut child = match child_result { Ok(c) => c, Err(e) => { - warn!(error = %e, "failed to spawn command"); + warn!(command = %program, error = %e, "{}", spawn_failure_message(&program, &e)); return None; } }; @@ -111,6 +112,14 @@ where } } +fn spawn_failure_message(program: &str, error: &std::io::Error) -> String { + if error.kind() == std::io::ErrorKind::NotFound { + format!("{program} not found; GPU metrics are unavailable on this host") + } else { + format!("failed to spawn {program}") + } +} + #[cfg(test)] #[path = "../tests/platform_common_tests.rs"] mod tests; diff --git a/services/gpu/src/tests/main_tests.rs b/services/gpu/src/tests/main_tests.rs index 5d11ae4..421b89a 100644 --- a/services/gpu/src/tests/main_tests.rs +++ b/services/gpu/src/tests/main_tests.rs @@ -169,6 +169,10 @@ fn config_load_and_record_snapshot_cover_main_paths() { max_file_bytes: 0, retain_files: 0, file_stem: "gpu-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; @@ -379,6 +383,10 @@ fn record_snapshot_handles_available_empty_samples() { max_file_bytes: 0, retain_files: 0, file_stem: "gpu-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; @@ -415,6 +423,10 @@ fn record_snapshot_returns_early_when_device_labels_disabled() { max_file_bytes: 0, retain_files: 0, file_stem: "gpu-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; @@ -460,6 +472,10 @@ fn record_snapshot_labeled_throttled_false_branch() { max_file_bytes: 0, retain_files: 0, file_stem: "gpu-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; diff --git a/services/gpu/src/tests/platform_common_tests.rs b/services/gpu/src/tests/platform_common_tests.rs index 11c063e..976fb28 100644 --- a/services/gpu/src/tests/platform_common_tests.rs +++ b/services/gpu/src/tests/platform_common_tests.rs @@ -1,4 +1,6 @@ -use super::{collect_snapshot, run_with_timeout, run_with_timeout_using_waiter}; +use super::{ + collect_snapshot, run_with_timeout, run_with_timeout_using_waiter, spawn_failure_message, +}; use std::fs; use std::process::Command; use std::sync::{Mutex, OnceLock}; @@ -68,6 +70,15 @@ fn collect_snapshot_returns_default_when_nvidia_smi_missing() { fs::remove_dir_all(&dir).expect("cleanup dir"); } +#[test] +fn spawn_failure_message_is_clear_for_missing_nvidia_smi() { + let err = std::io::Error::new(std::io::ErrorKind::NotFound, "No such file or directory"); + assert_eq!( + spawn_failure_message("nvidia-smi", &err), + "nvidia-smi not found; GPU metrics are unavailable on this host" + ); +} + #[test] fn collect_snapshot_handles_failed_and_malformed_output() { let _guard = env_lock().lock().expect("env lock"); diff --git a/services/mysql/README.md b/services/mysql/README.md index 5be4c79..ed083ca 100644 --- a/services/mysql/README.md +++ b/services/mysql/README.md @@ -23,9 +23,41 @@ Other key sections: - `metrics.include` / `metrics.exclude` - `storage.archive_*` +## Storage and archive +MySQL archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/mysql/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-mysql -- --config services/mysql/mysql.yaml ``` Use `--dump-snapshot` to print one snapshot as JSON. + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/mysql/mysql.yaml b/services/mysql/mysql.yaml index 90b2116..4a8279a 100644 --- a/services/mysql/mysql.yaml +++ b/services/mysql/mysql.yaml @@ -32,3 +32,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: mysql-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/mysql/src/main.rs b/services/mysql/src/main.rs index f4005c3..89b6137 100644 --- a/services/mysql/src/main.rs +++ b/services/mysql/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -337,6 +337,7 @@ fn run() -> Result<()> { let filter = PrefixFilter::new(cfg.metrics_include.clone(), cfg.metrics_exclude.clone()); let mut prev = PrevState::default(); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); let running = Arc::new(AtomicBool::new(true)); #[cfg(test)] @@ -548,6 +549,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } impl Config { @@ -630,6 +635,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "mysql-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) diff --git a/services/nfs-client/README.md b/services/nfs-client/README.md index ef61800..c3cfc3a 100644 --- a/services/nfs-client/README.md +++ b/services/nfs-client/README.md @@ -18,7 +18,39 @@ Main options: - `metrics.include` / `metrics.exclude` - `storage.archive_*` +## Storage and archive +NFS client archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/nfs-client/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-nfs-client -- --config services/nfs-client/nfs-client.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/nfs-client/nfs-client.yaml b/services/nfs-client/nfs-client.yaml index 7aaa976..397ed89 100644 --- a/services/nfs-client/nfs-client.yaml +++ b/services/nfs-client/nfs-client.yaml @@ -28,3 +28,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: nfs-client-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/nfs-client/src/main.rs b/services/nfs-client/src/main.rs index 1c747cb..d4bff28 100644 --- a/services/nfs-client/src/main.rs +++ b/services/nfs-client/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -274,6 +274,7 @@ fn run() -> Result<()> { let filter = PrefixFilter::new(cfg.metrics_include.clone(), cfg.metrics_exclude.clone()); let mut prev = PrevState::default(); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); let running = Arc::new(AtomicBool::new(true)); install_signal_handler(&running); @@ -437,6 +438,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } impl Config { @@ -517,6 +522,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "nfs-client-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) diff --git a/services/nfs-client/src/tests/platform_linux_tests.rs b/services/nfs-client/src/tests/platform_linux_tests.rs index 3efb57d..4c2f179 100644 --- a/services/nfs-client/src/tests/platform_linux_tests.rs +++ b/services/nfs-client/src/tests/platform_linux_tests.rs @@ -131,14 +131,7 @@ fn collect_rpc_stats_from_nfsstat_returns_none_when_spawn_fails() { #[test] fn collect_rpc_stats_from_nfsstat_parses_output_when_command_succeeds() { - let nanos = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .expect("clock") - .as_nanos(); - let dir = std::env::current_dir() - .expect("cwd") - .join("target") - .join(format!("nfsstat-success-{}-{nanos}", std::process::id())); + let dir = unique_temp_dir("nfsstat-success"); fs::create_dir_all(&dir).expect("mkdir"); let script = dir.join("fake-nfsstat.sh"); fs::write( diff --git a/services/nginx/README.md b/services/nginx/README.md index 5b113ef..a0db334 100644 --- a/services/nginx/README.md +++ b/services/nginx/README.md @@ -20,7 +20,39 @@ Common keys: - `metrics.include` / `metrics.exclude` - `storage.archive_*` +## Storage and archive +NGINX archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/nginx/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-nginx -- --config services/nginx/nginx.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/nginx/nginx.yaml b/services/nginx/nginx.yaml index 743f445..327e63a 100644 --- a/services/nginx/nginx.yaml +++ b/services/nginx/nginx.yaml @@ -28,3 +28,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: nginx-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/nginx/src/main.rs b/services/nginx/src/main.rs index 01e8a7b..a27069e 100644 --- a/services/nginx/src/main.rs +++ b/services/nginx/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::{Counter, Gauge}; use opentelemetry::KeyValue; @@ -328,6 +328,7 @@ fn run() -> Result<()> { let mut source_state = NginxConnectionState::Unknown; let mut export_state = ExportState::Pending; let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); #[cfg(test)] let mut iterations = 0u64; @@ -563,6 +564,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "nginx-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) @@ -630,6 +635,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[cfg(test)] diff --git a/services/nginx/src/tests/main_tests.rs b/services/nginx/src/tests/main_tests.rs index 41b53d8..6b034f7 100644 --- a/services/nginx/src/tests/main_tests.rs +++ b/services/nginx/src/tests/main_tests.rs @@ -431,6 +431,10 @@ fn config_shape_covers_all_fields() { max_file_bytes: 0, retain_files: 0, file_stem: "nginx-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; diff --git a/services/postgres/README.md b/services/postgres/README.md index 0a08eb7..580646d 100644 --- a/services/postgres/README.md +++ b/services/postgres/README.md @@ -21,7 +21,39 @@ Also configurable: - `metrics.include` / `metrics.exclude` - `storage.archive_*` +## Storage and archive +Postgres archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/postgres/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-postgres -- --config services/postgres/postgres.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/postgres/postgres.yaml b/services/postgres/postgres.yaml index 993c95e..14f7b07 100644 --- a/services/postgres/postgres.yaml +++ b/services/postgres/postgres.yaml @@ -28,3 +28,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: postgres-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/postgres/src/main.rs b/services/postgres/src/main.rs index 4234082..ca17709 100644 --- a/services/postgres/src/main.rs +++ b/services/postgres/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -259,6 +259,7 @@ fn run() -> Result<()> { let filter = PrefixFilter::new(cfg.metrics_include.clone(), cfg.metrics_exclude.clone()); let mut prev = PrevState::default(); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); let running = Arc::new(AtomicBool::new(true)); install_signal_handler(&running); @@ -503,6 +504,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } impl Config { @@ -581,6 +586,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "postgres-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) diff --git a/services/redis/README.md b/services/redis/README.md index e5c023e..f860499 100644 --- a/services/redis/README.md +++ b/services/redis/README.md @@ -23,7 +23,39 @@ Also configurable: - `metrics.include` / `metrics.exclude` - `storage.archive_*` +## Storage and archive +Redis archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/redis/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-redis -- --config services/redis/redis.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/redis/redis.yaml b/services/redis/redis.yaml index f9c77fc..da27d0f 100644 --- a/services/redis/redis.yaml +++ b/services/redis/redis.yaml @@ -22,3 +22,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: redis-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/redis/src/main.rs b/services/redis/src/main.rs index 39c3289..f74b0a0 100644 --- a/services/redis/src/main.rs +++ b/services/redis/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -279,6 +279,7 @@ fn run() -> Result<()> { let mut prev = PrevState::default(); let mut connection_state = RedisConnectionState::Unknown; let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); #[cfg(test)] let mut iterations = 0u64; @@ -618,6 +619,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "redis-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) @@ -688,6 +693,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[cfg(test)] diff --git a/services/redis/src/tests/main_tests.rs b/services/redis/src/tests/main_tests.rs index d7626f2..dc5e5b1 100644 --- a/services/redis/src/tests/main_tests.rs +++ b/services/redis/src/tests/main_tests.rs @@ -438,6 +438,10 @@ fn config_shape_covers_all_fields() { max_file_bytes: 0, retain_files: 0, file_stem: "redis-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; diff --git a/services/sensors/README.md b/services/sensors/README.md index 5904d8c..648f72f 100644 --- a/services/sensors/README.md +++ b/services/sensors/README.md @@ -20,9 +20,43 @@ Key options: - `storage.archive_*` ## Storage and archive -Snapshot NDJSON archival is controlled by `storage` and uses file rotation by size and file-count retention. +Sensors archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled`: enable/disable archive writer. +- `storage.archive_dir`: output directory. +- `storage.archive_max_file_bytes`: rotate when active file reaches this size. +- `storage.archive_retain_files`: number of rotated files to keep. +- `storage.archive_file_stem`: output file prefix. +- `storage.archive_format`: currently `parquet`. +- `storage.archive_mode`: + - `trend` (default): compact lossy window summaries (`min/max/avg/count/first/last`). + - `lossless`: full-fidelity row archives with columnar compression. + - `forensic`: compatibility mode, same behavior class as lossless row archival. +- `storage.archive_window_secs`: summary window size for `trend`. +- `storage.archive_compression`: parquet compression codec (`zstd`). + +Typical files: +- trend mode: `-trend.parquet` +- lossless mode: `-lossless.parquet` +- forensic mode: `-forensic.parquet` + +## Replay archives +Replay archives (all modes) to OTLP/Mimir: + +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/sensors/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` ## Run ```bash cargo run -p ojo-sensors -- --config services/sensors/sensors.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/sensors/sensors.yaml b/services/sensors/sensors.yaml index 134f03f..0f9c209 100644 --- a/services/sensors/sensors.yaml +++ b/services/sensors/sensors.yaml @@ -28,3 +28,8 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: sensor-snapshots + archive_format: parquet + # Modes: trend (compact lossy summaries), lossless (full-fidelity rows), forensic (compatibility row mode) + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/sensors/src/main.rs b/services/sensors/src/main.rs index 58186ed..862cbda 100644 --- a/services/sensors/src/main.rs +++ b/services/sensors/src/main.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, - OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -102,6 +102,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[derive(Clone, Debug, Default, Serialize)] @@ -279,6 +283,7 @@ fn run() -> Result<()> { let running = Arc::new(AtomicBool::new(true)); install_signal_handler(&running); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); let mut export_state = ExportState::Pending; let mut continue_running = true; @@ -498,6 +503,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "sensor-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) diff --git a/services/sensors/src/tests/main_tests.rs b/services/sensors/src/tests/main_tests.rs index 4ded102..7837484 100644 --- a/services/sensors/src/tests/main_tests.rs +++ b/services/sensors/src/tests/main_tests.rs @@ -372,6 +372,10 @@ fn record_snapshot_handles_available_without_voltages() { max_file_bytes: 0, retain_files: 0, file_stem: "sensor-snapshots".to_string(), + format: host_collectors::ArchiveFormat::Parquet, + mode: host_collectors::ArchiveMode::Trend, + window_secs: 60, + compression: host_collectors::ArchiveCompression::Zstd, }, once: true, }; diff --git a/services/syslog/README.md b/services/syslog/README.md index bbc408a..db20b0a 100644 --- a/services/syslog/README.md +++ b/services/syslog/README.md @@ -7,10 +7,12 @@ Syslog captures operating-system, process, and application log streams and expor - Optional file-watch ingestion from YAML-configured paths - Bounded in-memory buffer with backpressure-safe drop policy - Retry and reconnect logic for exporter outages -- Rotating NDJSON snapshot archive +- Rotating parquet archive modes (trend + lossless) ## Configuration -Primary config file: `services/syslog/syslog.yaml` +Primary config file: +- Linux: `services/syslog/syslog.yaml` +- Windows: `services/syslog/syslog.windows.yaml` Key sections: - `collection.poll_interval_secs` @@ -24,10 +26,49 @@ Key sections: - `storage.archive_dir` - `storage.archive_max_file_bytes` - `storage.archive_retain_files` +- `storage.archive_format` +- `storage.archive_mode` +- `storage.archive_window_secs` +- `storage.archive_compression` - `export.otlp.*` and `export.logs.*` - `metrics.include` / `metrics.exclude` +## Archive modes +- `trend` (default): compact lossy summaries for long-term trend analysis. +- `lossless`: preserves full log payload fidelity while using parquet + zstd compression. +- `forensic`: compatibility mode for row-level archival behavior. + +`archive_window_secs` applies to `trend` mode. Rotation and retention are controlled by `archive_max_file_bytes` and `archive_retain_files`. + +## Replay to Mimir +Use the replay utility to push archived parquet files (trend/lossless/forensic) into OTLP or remote-write endpoints: + +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/syslog/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + +Remote write example: + +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/syslog/data \ + --endpoint http://localhost:4320/mimir/api/v1/push \ + --protocol remote-write +``` + ## Run ```bash cargo run -p ojo-syslog -- --config services/syslog/syslog.yaml ``` + +On Windows, the default config path resolves to `services/syslog/syslog.windows.yaml`, which keeps the ETW-based collector enabled without Linux-only `/var/log` file watches. + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/syslog/src/main.rs b/services/syslog/src/main.rs index e7e8a5c..b4db319 100644 --- a/services/syslog/src/main.rs +++ b/services/syslog/src/main.rs @@ -1,6 +1,8 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, OtlpSettings, PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, ArchiveCompression, ArchiveFormat, + ArchiveMode, ArchiveStorageConfig, ArchiveWriter, JsonArchiveWriter, OtlpSettings, + PrefixFilter, }; use opentelemetry::metrics::{Counter, Gauge}; use opentelemetry::KeyValue; @@ -8,8 +10,8 @@ use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::collections::{BTreeMap, VecDeque}; use std::env; -use std::fs::{self as stdfs, OpenOptions}; -use std::io::Write; +#[cfg(test)] +use std::fs::{self as stdfs}; use std::path::Path; use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -48,6 +50,10 @@ struct Config { archive_dir: String, archive_max_file_bytes: u64, archive_retain_files: usize, + archive_format: ArchiveFormat, + archive_mode: ArchiveMode, + archive_window_secs: u64, + archive_compression: ArchiveCompression, once: bool, } @@ -316,23 +322,45 @@ impl OtlpLogExporter { } } -#[derive(Clone, Debug)] struct ArchivePipeline { enabled: bool, - dir: String, + #[cfg(test)] max_file_bytes: u64, + #[cfg(test)] retain_files: usize, + writer: JsonArchiveWriter, + total_events: u64, + total_bytes: u64, healthy: bool, last_error: Option, } impl ArchivePipeline { fn from_config(cfg: &Config) -> Self { + let writer_cfg = ArchiveStorageConfig { + enabled: cfg.archive_enabled, + archive_dir: cfg.archive_dir.clone(), + max_file_bytes: cfg.archive_max_file_bytes, + retain_files: cfg.archive_retain_files, + file_stem: "syslog".to_string(), + format: cfg.archive_format.clone(), + mode: cfg.archive_mode.clone(), + window_secs: cfg.archive_window_secs, + compression: cfg.archive_compression.clone(), + }; Self { enabled: cfg.archive_enabled, - dir: cfg.archive_dir.clone(), + #[cfg(test)] max_file_bytes: cfg.archive_max_file_bytes, + #[cfg(test)] retain_files: cfg.archive_retain_files, + writer: { + let mut writer = JsonArchiveWriter::from_config(&writer_cfg); + writer.set_default_identity(&cfg.service_name, &cfg.instance_id); + writer + }, + total_events: 0, + total_bytes: 0, healthy: true, last_error: None, } @@ -343,41 +371,32 @@ impl ArchivePipeline { return; } - if let Err(err) = self.write_batch_impl(batch) { - self.healthy = false; - self.last_error = Some(err.to_string()); - warn!(error = %err, "archive write failed for syslog batch"); - } else { - self.healthy = true; - self.last_error = None; + self.write_batch_impl(batch); + if !self.healthy { + let detail = self + .last_error + .clone() + .unwrap_or_else(|| "unknown archive writer error".to_string()); + warn!(error = %detail, "archive write failed for syslog batch"); } } - fn write_batch_impl(&mut self, batch: &[LogRecord]) -> Result<()> { - stdfs::create_dir_all(&self.dir) - .with_context(|| format!("failed to create archive dir '{}'", self.dir))?; - - let path = format!("{}/syslog-records.ndjson", self.dir); - self.rotate_if_needed(&path)?; - - let mut file = OpenOptions::new() - .create(true) - .append(true) - .open(&path) - .with_context(|| format!("failed to open archive '{}'", path))?; - - for record in batch { - let line = serde_json::json!(record).to_string(); - file.write_all(line.as_bytes()) - .context("failed to write archive log line")?; - file.write_all(b"\n") - .context("failed to write archive newline")?; - } - - self.prune_rotated_files(&path)?; - Ok(()) + fn write_batch_impl(&mut self, batch: &[LogRecord]) { + let values = batch + .iter() + .map(|record| { + serde_json::to_value(record).expect("LogRecord serialization should not fail") + }) + .collect::>(); + self.writer.write_log_batch(&values); + self.writer.flush(); + self.total_events = self.writer.total_records; + self.total_bytes = self.writer.total_bytes; + self.healthy = self.writer.healthy; + self.last_error = self.writer.last_error.clone(); } + #[cfg(test)] fn rotate_if_needed(&self, path: &str) -> Result<()> { let metadata = match stdfs::metadata(path) { Ok(md) => md, @@ -398,6 +417,7 @@ impl ArchivePipeline { Ok(()) } + #[cfg(test)] fn prune_rotated_files(&self, prefix: &str) -> Result<()> { if self.retain_files == 0 { return Ok(()); @@ -625,9 +645,7 @@ impl Config { .find(|p| p[0] == "--config") .map(|p| p[1].clone()) .or_else(|| env::var("OJO_SYSLOG_CONFIG").ok()) - .unwrap_or_else(|| { - resolve_default_config_path("syslog.yaml", "services/syslog/syslog.yaml") - }); + .unwrap_or_else(default_config_path); let file_cfg = load_yaml_config_file(&config_path)?; let service = file_cfg.service.unwrap_or_default(); @@ -685,11 +703,27 @@ impl Config { .unwrap_or_else(|| "services/syslog/data".to_string()), archive_max_file_bytes: storage.archive_max_file_bytes.unwrap_or(64 * 1024 * 1024), archive_retain_files: storage.archive_retain_files.unwrap_or(8), + archive_format: ArchiveFormat::parse(storage.archive_format.as_deref()), + archive_mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + archive_window_secs: storage.archive_window_secs.unwrap_or(60), + archive_compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), once, }) } } +fn default_config_path() -> String { + #[cfg(target_os = "windows")] + { + resolve_default_config_path("syslog.windows.yaml", "services/syslog/syslog.windows.yaml") + } + + #[cfg(not(target_os = "windows"))] + { + resolve_default_config_path("syslog.yaml", "services/syslog/syslog.yaml") + } +} + fn advance_export_state(current: ExportState, export_succeeded: bool) -> (ExportState, FlushEvent) { if export_succeeded { let event = match current { @@ -1044,6 +1078,10 @@ struct StorageSection { archive_dir: Option, archive_max_file_bytes: Option, archive_retain_files: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[derive(Clone, Debug, Default, Deserialize)] diff --git a/services/syslog/src/platform/common.rs b/services/syslog/src/platform/common.rs index 8701f46..edfab6d 100644 --- a/services/syslog/src/platform/common.rs +++ b/services/syslog/src/platform/common.rs @@ -1,31 +1,35 @@ -use std::process::{Child, Command, Stdio}; +use std::io::Read; +use std::process::{Child, Command, Output, Stdio}; +use std::thread; use std::time::{Duration, Instant}; use tracing::warn; +#[cfg(any(test, not(coverage)))] const CMD_TIMEOUT: Duration = Duration::from_secs(12); -pub(super) fn run_command_with_timeout( - command: &str, - args: &[&str], -) -> Option { +#[cfg(any(test, not(coverage)))] +pub(super) fn run_command_with_timeout(command: &str, args: &[&str]) -> Option { let mut cmd = Command::new(command); cmd.args(args); run_with_timeout(cmd, CMD_TIMEOUT) } -fn run_with_timeout(cmd: Command, timeout: Duration) -> Option { +#[cfg(any(test, not(coverage)))] +fn run_with_timeout(cmd: Command, timeout: Duration) -> Option { run_with_timeout_using_waiter(cmd, timeout, wait_for_child) } +#[cfg(any(test, not(coverage)))] fn wait_for_child(child: &mut Child) -> std::io::Result> { child.try_wait() } +#[cfg(any(test, not(coverage)))] fn run_with_timeout_using_waiter( mut cmd: Command, timeout: Duration, mut waiter: W, -) -> Option +) -> Option where W: FnMut(&mut Child) -> std::io::Result>, { @@ -40,14 +44,27 @@ where } }; + let stdout = child.stdout.take().map(spawn_output_reader); + let stderr = child.stderr.take().map(spawn_output_reader); + let start = Instant::now(); loop { match waiter(&mut child) { - Ok(Some(_)) => return child.wait_with_output().ok(), + Ok(Some(status)) => { + let stdout = join_output_reader(stdout); + let stderr = join_output_reader(stderr); + return Some(Output { + status, + stdout, + stderr, + }); + } Ok(None) => { if start.elapsed() > timeout { let _ = child.kill(); let _ = child.wait(); + let _ = join_output_reader(stdout); + let _ = join_output_reader(stderr); warn!("command timed out after {:?}", timeout); return None; } @@ -56,12 +73,31 @@ where Err(e) => { warn!(error = %e, "error waiting for command"); let _ = child.kill(); + let _ = child.wait(); + let _ = join_output_reader(stdout); + let _ = join_output_reader(stderr); return None; } } } } +#[cfg(any(test, not(coverage)))] +fn spawn_output_reader(mut reader: impl Read + Send + 'static) -> thread::JoinHandle> { + thread::spawn(move || { + let mut buffer = Vec::new(); + let _ = reader.read_to_end(&mut buffer); + buffer + }) +} + +#[cfg(any(test, not(coverage)))] +fn join_output_reader(handle: Option>>) -> Vec { + handle + .map(|handle| handle.join().unwrap_or_default()) + .unwrap_or_default() +} + #[cfg(test)] mod tests { use super::*; @@ -80,6 +116,13 @@ mod tests { assert_eq!(String::from_utf8_lossy(&out.stdout), "ok"); } + #[test] + fn run_command_with_timeout_handles_large_output_without_deadlocking() { + let out = run_command_with_timeout("sh", &["-c", "yes x | head -c 80000"]).expect("output"); + assert!(out.status.success()); + assert!(out.stdout.len() >= 80000); + } + #[test] fn run_with_timeout_using_waiter_covers_success_timeout_and_wait_error() { let mut success_cmd = Command::new("sh"); diff --git a/services/syslog/src/platform/coverage.rs b/services/syslog/src/platform/coverage.rs index 1068637..f05b2ff 100644 --- a/services/syslog/src/platform/coverage.rs +++ b/services/syslog/src/platform/coverage.rs @@ -1,8 +1,17 @@ -use crate::LogRecord; +use crate::{normalize_severity, now_unix_nanos, sanitize_ascii_line, LogRecord}; use super::{PlatformCollection, PlatformConfig, PlatformSnapshot}; -pub(crate) fn collect(_cfg: &PlatformConfig) -> PlatformCollection { +pub(crate) fn collect(cfg: &PlatformConfig) -> PlatformCollection { + let _ = cfg.max_lines_per_source; + let _ = cfg.max_message_bytes; + for watch in &cfg.watch_files { + let _ = (&watch.name, &watch.path, watch.source); + } + let _ = now_unix_nanos(); + let _ = sanitize_ascii_line("coverage", 8); + let _ = normalize_severity("INFO"); + PlatformCollection { snapshot: PlatformSnapshot { available: true, diff --git a/services/syslog/src/platform/linux.rs b/services/syslog/src/platform/linux.rs index 1050048..f5ad89a 100644 --- a/services/syslog/src/platform/linux.rs +++ b/services/syslog/src/platform/linux.rs @@ -1,4 +1,5 @@ use crate::{normalize_severity, now_unix_nanos, sanitize_ascii_line, LogRecord}; +use serde_json::Value; use super::LinuxSourceSnapshot; @@ -9,13 +10,15 @@ pub(super) fn collect_linux( let mut snapshot = LinuxSourceSnapshot::default(); let mut records = Vec::new(); + let max_lines = max_lines_per_source.to_string(); if let Some(output) = super::common::run_command_with_timeout( "journalctl", &[ "-n", - &max_lines_per_source.to_string(), + max_lines.as_str(), "--no-pager", - "--output=short-iso", + "--output=json", + "--output-fields=MESSAGE", ], ) { if output.status.success() { @@ -44,22 +47,35 @@ pub(super) fn collect_linux( } fn collect_journal_records(text: &str, max_message_bytes: usize) -> Vec { - let mut records = Vec::new(); - for line in text.lines().filter(|line| !line.trim().is_empty()) { - let body = sanitize_ascii_line(line, max_message_bytes); - if body.is_empty() { - continue; + let body = super::join_snapshot_bodies( + text.lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| journal_record_body(line, max_message_bytes)), + max_message_bytes, + ); + + if body.is_empty() { + return Vec::new(); + } + + vec![LogRecord { + observed_time_unix_nano: now_unix_nanos(), + severity_text: "INFO".to_string(), + body, + source: "journald".to_string(), + stream: "system".to_string(), + watch_target: "journald".to_string(), + }] +} + +fn journal_record_body(line: &str, max_message_bytes: usize) -> String { + if let Ok(entry) = serde_json::from_str::(line) { + if let Some(message) = entry.get("MESSAGE").and_then(|value| value.as_str()) { + return sanitize_ascii_line(message, max_message_bytes); } - records.push(LogRecord { - observed_time_unix_nano: now_unix_nanos(), - severity_text: "INFO".to_string(), - body, - source: "journald".to_string(), - stream: "system".to_string(), - watch_target: "journald".to_string(), - }); } - records + + sanitize_ascii_line(line, max_message_bytes) } fn collect_dmesg_records( @@ -67,7 +83,6 @@ fn collect_dmesg_records( max_lines_per_source: usize, max_message_bytes: usize, ) -> Vec { - let mut records = Vec::new(); let mut lines = text .lines() .filter(|line| !line.trim().is_empty()) @@ -76,31 +91,49 @@ fn collect_dmesg_records( let start = lines.len() - max_lines_per_source; lines = lines.split_off(start); } - for line in lines { - let lower = line.to_ascii_lowercase(); - let severity = if lower.contains("error") || lower.contains("fail") { - "ERROR" - } else if lower.contains("warn") { - "WARN" - } else if lower.contains("debug") { - "DEBUG" - } else { - "INFO" - }; - let body = sanitize_ascii_line(line, max_message_bytes); - if body.is_empty() { - continue; - } - records.push(LogRecord { - observed_time_unix_nano: now_unix_nanos(), - severity_text: normalize_severity(severity), - body, - source: "kernel".to_string(), - stream: "dmesg".to_string(), - watch_target: "dmesg".to_string(), - }); + + let mut severity = "INFO"; + let body = super::join_snapshot_bodies( + lines.into_iter().filter_map(|line| { + let lower = line.to_ascii_lowercase(); + let line_severity = if lower.contains("error") || lower.contains("fail") { + "ERROR" + } else if lower.contains("warn") { + "WARN" + } else if lower.contains("debug") { + "DEBUG" + } else { + "INFO" + }; + severity = match (severity, line_severity) { + ("ERROR", _) | (_, "ERROR") => "ERROR", + ("WARN", _) | (_, "WARN") => "WARN", + ("DEBUG", _) | (_, "DEBUG") => "DEBUG", + _ => "INFO", + }; + + let body = sanitize_ascii_line(line, max_message_bytes); + if body.is_empty() { + None + } else { + Some(body) + } + }), + max_message_bytes, + ); + + if body.is_empty() { + return Vec::new(); } - records + + vec![LogRecord { + observed_time_unix_nano: now_unix_nanos(), + severity_text: normalize_severity(severity), + body, + source: "kernel".to_string(), + stream: "dmesg".to_string(), + watch_target: "dmesg".to_string(), + }] } #[cfg(test)] @@ -111,20 +144,38 @@ mod tests { fn collect_journal_records_skips_empty_and_sanitizes() { let text = "line-1\n\nline-2\u{2603}\n"; let records = collect_journal_records(text, 16); - assert_eq!(records.len(), 2); + assert_eq!(records.len(), 1); assert_eq!(records[0].source, "journald"); - assert!(records[1].body.contains('?')); + assert_eq!(records[0].body, "line-1\nline-2?"); + } + + #[test] + fn collect_journal_records_keeps_multiline_journal_messages_together() { + let text = concat!( + r#"{"MESSAGE":"Value: 40.000000\nTimestamp: 2026-04-24 04:03:22.813 +0000 UTC\nStartTimestamp: 1970-01-01 00:00:00 +0000 UTC","PRIORITY":6}"#, + "\n", + r#"{"MESSAGE":"single line","PRIORITY":6}"#, + "\n" + ); + + let records = collect_journal_records(text, 512); + + assert_eq!(records.len(), 1); + assert_eq!(records[0].body.lines().count(), 4); + assert!(records[0].body.contains("Value: 40.000000")); + assert!(records[0].body.contains("single line")); } #[test] fn collect_dmesg_records_applies_tail_and_severity_mapping() { let text = "debug boot\nwarn disk\nerror fail\ninfo done\n"; let records = collect_dmesg_records(text, 3, 128); - assert_eq!(records.len(), 3); - assert_eq!(records[0].severity_text, "WARN"); - assert_eq!(records[1].severity_text, "ERROR"); - assert_eq!(records[2].severity_text, "INFO"); - assert!(records.iter().all(|r| r.source == "kernel")); - assert!(records.iter().all(|r| r.stream == "dmesg")); + assert_eq!(records.len(), 1); + assert_eq!(records[0].severity_text, "ERROR"); + assert!(records[0].body.contains("warn disk")); + assert!(records[0].body.contains("error fail")); + assert!(records[0].body.contains("info done")); + assert_eq!(records[0].source, "kernel"); + assert_eq!(records[0].stream, "dmesg"); } } diff --git a/services/syslog/src/platform/mod.rs b/services/syslog/src/platform/mod.rs index 95cfc2c..730058d 100644 --- a/services/syslog/src/platform/mod.rs +++ b/services/syslog/src/platform/mod.rs @@ -43,7 +43,7 @@ pub(crate) struct PlatformCollection { pub(crate) records: Vec, } -#[cfg(target_os = "linux")] +#[cfg(all(not(coverage), target_os = "linux"))] #[derive(Clone, Copy, Debug, Default)] struct LinuxSourceSnapshot { available: bool, @@ -51,7 +51,7 @@ struct LinuxSourceSnapshot { dmesg_available: bool, } -#[cfg(target_os = "windows")] +#[cfg(all(not(coverage), target_os = "windows"))] #[derive(Clone, Copy, Debug, Default)] struct WindowsSourceSnapshot { available: bool, @@ -59,6 +59,7 @@ struct WindowsSourceSnapshot { application_logs_available: bool, } +#[cfg(any(test, not(coverage)))] fn file_offsets() -> &'static Mutex> { static OFFSETS: OnceLock>> = OnceLock::new(); OFFSETS.get_or_init(|| Mutex::new(BTreeMap::new())) @@ -147,6 +148,7 @@ fn collect_fallback(cfg: &PlatformConfig) -> PlatformCollection { } } +#[cfg(any(test, not(coverage)))] struct WatchedFileResult { records: Vec, active_targets: u64, @@ -155,6 +157,36 @@ struct WatchedFileResult { collection_errors: u64, } +#[cfg(any(test, not(coverage)))] +fn join_snapshot_bodies(bodies: I, max_message_bytes: usize) -> String +where + I: IntoIterator, +{ + let mut body = String::new(); + + for part in bodies { + if part.is_empty() { + continue; + } + + if body.is_empty() { + body.push_str(&part); + continue; + } + + let next_len = body.len() + 1 + part.len(); + if next_len > max_message_bytes { + break; + } + + body.push('\n'); + body.push_str(&part); + } + + body +} + +#[cfg(any(test, not(coverage)))] fn collect_watched_file_records(cfg: &PlatformConfig) -> WatchedFileResult { let mut records = Vec::new(); let mut active_targets = 0u64; @@ -182,30 +214,37 @@ fn collect_watched_file_records(cfg: &PlatformConfig) -> WatchedFileResult { let lines = content.lines().collect::>(); let consumed = offsets.get(&watch.path).copied().unwrap_or(0) as usize; let start = consumed.min(lines.len()); - let mut produced = 0usize; + let body = join_snapshot_bodies( + lines + .iter() + .skip(start) + .take(cfg.max_lines_per_source) + .filter_map(|line| { + let body = sanitize_ascii_line(line, cfg.max_message_bytes); + if body.is_empty() { + None + } else { + Some(body) + } + }), + cfg.max_message_bytes, + ); - for line in lines.iter().skip(start).take(cfg.max_lines_per_source) { - let body = sanitize_ascii_line(line, cfg.max_message_bytes); - if body.is_empty() { - continue; - } + offsets.insert(watch.path.clone(), lines.len() as u64); + if !body.is_empty() { let source = match watch.source { WatchSource::Application => "application", WatchSource::Process => "process", }; + let watch_target = sanitize_watch_target_name(&watch.name); records.push(LogRecord { observed_time_unix_nano: now_unix_nanos(), severity_text: "INFO".to_string(), body, source: source.to_string(), stream: "file".to_string(), - watch_target: sanitize_watch_target_name(&watch.name), + watch_target, }); - produced += 1; - } - - offsets.insert(watch.path.clone(), lines.len() as u64); - if produced > 0 { active_targets = active_targets.saturating_add(1); match watch.source { WatchSource::Application => application_logs_available = true, @@ -223,6 +262,7 @@ fn collect_watched_file_records(cfg: &PlatformConfig) -> WatchedFileResult { } } +#[cfg(any(test, not(coverage)))] fn sanitize_watch_target_name(name: &str) -> String { let mut out = String::new(); for ch in name.chars() { diff --git a/services/syslog/src/platform/windows.rs b/services/syslog/src/platform/windows.rs index fa608b1..05a7d18 100644 --- a/services/syslog/src/platform/windows.rs +++ b/services/syslog/src/platform/windows.rs @@ -19,31 +19,12 @@ pub(super) fn collect_windows( snapshot.etw_available = true; snapshot.application_logs_available = true; let text = String::from_utf8_lossy(&output.stdout); - for line in text.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with("Event[") { - continue; - } - let body = sanitize_ascii_line(trimmed, max_message_bytes); - if body.is_empty() { - continue; - } - let severity = if body.contains("Level: Error") { - "ERROR" - } else if body.contains("Level: Warning") { - "WARN" - } else { - "INFO" - }; - records.push(LogRecord { - observed_time_unix_nano: now_unix_nanos(), - severity_text: normalize_severity(severity), - body, - source: "windows_eventlog".to_string(), - stream: "system".to_string(), - watch_target: "windows-system".to_string(), - }); - } + records.extend(collect_eventlog_records( + &text, + max_message_bytes, + "system", + "windows-system", + )); } } @@ -61,34 +42,134 @@ pub(super) fn collect_windows( snapshot.etw_available = true; snapshot.application_logs_available = true; let text = String::from_utf8_lossy(&output.stdout); - for line in text.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with("Event[") { - continue; - } - let body = sanitize_ascii_line(trimmed, max_message_bytes); - if body.is_empty() { - continue; - } - let severity = if body.contains("Level: Error") { - "ERROR" - } else if body.contains("Level: Warning") { - "WARN" - } else { - "INFO" - }; - records.push(LogRecord { - observed_time_unix_nano: now_unix_nanos(), - severity_text: normalize_severity(severity), - body, - source: "windows_eventlog".to_string(), - stream: "application".to_string(), - watch_target: "windows-application".to_string(), - }); - } + records.extend(collect_eventlog_records( + &text, + max_message_bytes, + "application", + "windows-application", + )); } } snapshot.available = snapshot.etw_available; (snapshot, records) } + +fn collect_eventlog_records( + text: &str, + max_message_bytes: usize, + stream: &str, + watch_target: &str, +) -> Vec { + let mut records = Vec::new(); + let mut current_event_lines = Vec::new(); + let mut in_event = false; + + for raw_line in text.lines() { + let trimmed = raw_line.trim(); + if trimmed.starts_with("Event[") { + if in_event { + push_event_record( + ¤t_event_lines, + max_message_bytes, + stream, + watch_target, + &mut records, + ); + current_event_lines.clear(); + } + in_event = true; + current_event_lines.push(raw_line.to_string()); + continue; + } + + if !in_event { + continue; + } + + current_event_lines.push(raw_line.to_string()); + } + + if in_event { + push_event_record( + ¤t_event_lines, + max_message_bytes, + stream, + watch_target, + &mut records, + ); + } + + records +} + +fn push_event_record( + event_lines: &[String], + max_message_bytes: usize, + stream: &str, + watch_target: &str, + records: &mut Vec, +) { + if event_lines.is_empty() { + return; + } + + let body = sanitize_ascii_line(&event_lines.join("\n"), max_message_bytes); + if body.is_empty() { + return; + } + + let severity = if event_lines.iter().any(|line| line.contains("Level: Error")) { + "ERROR" + } else if event_lines + .iter() + .any(|line| line.contains("Level: Warning")) + { + "WARN" + } else { + "INFO" + }; + + records.push(LogRecord { + observed_time_unix_nano: now_unix_nanos(), + severity_text: normalize_severity(severity), + body, + source: "windows_eventlog".to_string(), + stream: stream.to_string(), + watch_target: watch_target.to_string(), + }); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn collect_eventlog_records_groups_event_blocks() { + let text = concat!( + "Event[0]:\n", + " Log Name: System\n", + " Level: Warning\n", + " Message: First line\n", + " Second line\n", + "\n", + "Event[1]:\n", + " Log Name: Application\n", + " Level: Error\n", + " Message: Third line\n", + ); + + let records = collect_eventlog_records(text, 512, "system", "windows-system"); + assert_eq!(records.len(), 2); + assert_eq!(records[0].source, "windows_eventlog"); + assert_eq!(records[0].stream, "system"); + assert_eq!(records[0].watch_target, "windows-system"); + assert!(records[0].body.contains("Event[0]:")); + assert!(records[0].body.contains("First line")); + assert!(records[0].body.lines().count() >= 4); + assert_eq!(records[0].severity_text, "WARN"); + assert_eq!(records[1].severity_text, "ERROR"); + assert!(records[1].body.contains("Event[1]:")); + assert!(records[1].body.contains("Third line")); + } +} diff --git a/services/syslog/src/tests/main_tests.rs b/services/syslog/src/tests/main_tests.rs index f53d251..53c725b 100644 --- a/services/syslog/src/tests/main_tests.rs +++ b/services/syslog/src/tests/main_tests.rs @@ -5,7 +5,7 @@ use crate::{ record_snapshot, resolve_default_config_path, sanitize_ascii_line, ArchivePipeline, Config, ExportState, FlushEvent, Instruments, LogBuffer, LogRecord, OtlpLogExporter, RuntimeSnapshot, }; -use host_collectors::PrefixFilter; +use host_collectors::{ArchiveCompression, ArchiveFormat, ArchiveMode, PrefixFilter}; use std::fs; use std::io::{Read, Write}; use std::net::TcpListener; @@ -136,6 +136,39 @@ fn resolve_default_config_path_prefers_existing_local() { assert_eq!(selected, "fallback.yaml"); } +#[cfg(target_os = "windows")] +#[test] +fn resolve_default_config_path_prefers_windows_syslog_default() { + let selected = + resolve_default_config_path("syslog.windows.yaml", "services/syslog/syslog.windows.yaml"); + assert_eq!(selected, "syslog.windows.yaml"); +} + +#[cfg(target_os = "windows")] +#[test] +fn load_windows_companion_config_parses() { + let config_path = concat!(env!("CARGO_MANIFEST_DIR"), "/syslog.windows.yaml"); + let file_cfg = load_yaml_config_file(config_path).expect("windows config"); + + let service = file_cfg.service.expect("service section"); + assert_eq!(service.name.as_deref(), Some("ojo-syslog")); + assert_eq!(service.instance_id.as_deref(), Some("syslog-windows")); + + let collection = file_cfg.collection.expect("collection section"); + assert_eq!(collection.poll_interval_secs, Some(5)); + assert_eq!(collection.max_lines_per_source, Some(200)); + assert_eq!(collection.max_message_bytes, Some(4096)); + + let watch = file_cfg.watch.expect("watch section"); + assert!(watch.files.unwrap_or_default().is_empty()); + + let export = file_cfg.export.expect("export section"); + assert_eq!( + export.otlp.expect("otlp section").endpoint.as_deref(), + Some("http://192.168.0.214:4355/v1/metrics") + ); +} + #[test] fn load_yaml_config_file_handles_missing_empty_invalid() { let missing = unique_temp_path("syslog-missing.yaml"); @@ -243,6 +276,10 @@ fn test_config_with_logs_endpoint(logs_endpoint: String) -> Config { archive_dir: "".to_string(), archive_max_file_bytes: 1024, archive_retain_files: 2, + archive_format: ArchiveFormat::Parquet, + archive_mode: ArchiveMode::Trend, + archive_window_secs: 60, + archive_compression: ArchiveCompression::Zstd, once: true, } } @@ -419,7 +456,7 @@ fn archive_pipeline_writes_and_rotates_records() { cfg.archive_retain_files = 2; let mut archive = ArchivePipeline::from_config(&cfg); - let path = dir.join("syslog-records.ndjson"); + let path = dir.join("syslog-trend.parquet"); archive.write_batch(&[sample_record("first")]); archive.write_batch(&[sample_record("second")]); @@ -451,6 +488,36 @@ fn archive_pipeline_marks_unhealthy_on_invalid_dir() { fs::remove_dir_all(&parent).expect("cleanup"); } +#[test] +fn archive_pipeline_rotate_if_needed_covers_missing_and_oversized_paths() { + let dir = unique_temp_path("syslog-archive-rotate-branches"); + fs::create_dir_all(&dir).expect("create archive dir"); + let mut cfg = test_config_with_logs_endpoint("http://127.0.0.1:1/v1/logs".to_string()); + cfg.archive_dir = dir.to_string_lossy().to_string(); + cfg.archive_max_file_bytes = 8; + cfg.archive_retain_files = 2; + let archive = ArchivePipeline::from_config(&cfg); + + let missing_path = dir.join("missing.parquet"); + archive + .rotate_if_needed(missing_path.to_string_lossy().as_ref()) + .expect("missing file should be a no-op"); + + let path = dir.join("syslog-trend.parquet"); + fs::write(&path, b"this file is oversized").expect("seed oversized archive"); + fs::write(format!("{}.1", path.to_string_lossy()), b"old1").expect("seed .1"); + fs::write(format!("{}.2", path.to_string_lossy()), b"old2").expect("seed .2"); + + archive + .rotate_if_needed(path.to_string_lossy().as_ref()) + .expect("oversized file should rotate"); + + assert!(std::path::Path::new(&format!("{}.1", path.to_string_lossy())).exists()); + assert!(std::path::Path::new(&format!("{}.2", path.to_string_lossy())).exists()); + + fs::remove_dir_all(&dir).expect("cleanup archive dir"); +} + #[test] fn export_buffered_logs_handles_success_and_error() { let listener = TcpListener::bind("127.0.0.1:0").expect("bind test server"); diff --git a/services/syslog/src/tests/platform_tests.rs b/services/syslog/src/tests/platform_tests.rs index 6ef2558..c92ec22 100644 --- a/services/syslog/src/tests/platform_tests.rs +++ b/services/syslog/src/tests/platform_tests.rs @@ -1,9 +1,9 @@ use crate::{WatchSource, WatchedFileConfig}; -use crate::platform::{ - file_offsets, sanitize_watch_target_name, PlatformCollection, PlatformConfig, -}; +use crate::platform::{file_offsets, sanitize_watch_target_name, PlatformConfig}; use std::fs; +use std::fs::OpenOptions; +use std::io::Write; use std::sync::{Mutex, OnceLock}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -16,29 +16,6 @@ fn reset_offsets() { file_offsets().lock().expect("offset lock").clear(); } -fn collect_for_test(cfg: &PlatformConfig) -> PlatformCollection { - #[cfg(coverage)] - { - let watched = crate::platform::collect_watched_file_records(cfg); - return PlatformCollection { - snapshot: crate::platform::PlatformSnapshot { - available: watched.active_targets > 0, - process_logs_available: watched.process_logs_available, - application_logs_available: watched.application_logs_available, - file_watch_targets_active: watched.active_targets, - collection_errors: watched.collection_errors, - ..crate::platform::PlatformSnapshot::default() - }, - records: watched.records, - }; - } - - #[cfg(not(coverage))] - { - crate::platform::collect(cfg) - } -} - fn unique_temp_path(name: &str) -> std::path::PathBuf { let nanos = SystemTime::now() .duration_since(UNIX_EPOCH) @@ -61,15 +38,20 @@ fn sanitize_watch_target_name_normalizes_symbols() { } #[test] -fn collect_reads_incremental_lines_from_watched_files() { +fn collect_groups_incremental_lines_from_watched_files() { let _guard = offsets_test_lock().lock().expect("offset test lock"); reset_offsets(); let watched = unique_temp_path("watch.log"); - fs::write(&watched, "line1\nline2\n").expect("seed file"); + let initial = (1..=20) + .map(|index| format!("line-{index:02}-abcdefghij")) + .collect::>() + .join("\n") + + "\n"; + fs::write(&watched, &initial).expect("seed file"); let cfg = PlatformConfig { max_lines_per_source: 10, - max_message_bytes: 256, + max_message_bytes: 64, watch_files: vec![WatchedFileConfig { name: "app-log".to_string(), path: watched.to_string_lossy().to_string(), @@ -77,13 +59,35 @@ fn collect_reads_incremental_lines_from_watched_files() { }], }; - let first = collect_for_test(&cfg); - assert!(first.records.iter().any(|r| r.body.contains("line1"))); - assert_eq!(first.snapshot.file_watch_targets_active, 1); - - fs::write(&watched, "line1\nline2\nline3\n").expect("append file"); - let second = collect_for_test(&cfg); - assert!(second.records.iter().any(|r| r.body.contains("line3"))); + let first = crate::platform::collect_watched_file_records(&cfg); + let first_watch_records = first + .records + .iter() + .filter(|record| record.source == "application" && record.stream == "file") + .collect::>(); + assert_eq!(first_watch_records.len(), 1, "records: {:?}", first.records); + assert!(first_watch_records[0].body.contains("line-01-abcdefghij")); + assert_eq!(first.active_targets, 1); + + let mut append = OpenOptions::new() + .append(true) + .open(&watched) + .expect("open append file"); + writeln!(append, "line-21-abcdefghij").expect("append line 21"); + writeln!(append, "line-22-abcdefghij").expect("append line 22"); + let second = crate::platform::collect_watched_file_records(&cfg); + let second_watch_records = second + .records + .iter() + .filter(|record| record.source == "application" && record.stream == "file") + .collect::>(); + assert_eq!( + second_watch_records.len(), + 1, + "records: {:?}", + second.records + ); + assert!(second_watch_records[0].body.contains("line-21-abcdefghij")); fs::remove_file(&watched).expect("cleanup"); } @@ -102,8 +106,8 @@ fn collect_handles_missing_watch_files() { }], }; - let result = collect_for_test(&cfg); - assert_eq!(result.snapshot.file_watch_targets_active, 0); + let result = crate::platform::collect_watched_file_records(&cfg); + assert_eq!(result.active_targets, 0); } #[test] @@ -124,8 +128,8 @@ fn collect_counts_error_when_watch_target_is_unreadable_directory() { }], }; - let result = collect_for_test(&cfg); - assert!(result.snapshot.collection_errors >= 1); + let result = crate::platform::collect_watched_file_records(&cfg); + assert!(result.collection_errors >= 1); fs::remove_dir_all(&dir).expect("cleanup"); } @@ -148,11 +152,12 @@ fn collect_skips_watch_lines_that_sanitize_to_empty() { }], }; - let result = collect_for_test(&cfg); - assert!(!result.records.iter().any(|record| { - record.source == "application" && record.watch_target == "empty_line_watch" - })); - assert_eq!(result.snapshot.file_watch_targets_active, 0); + let result = crate::platform::collect_watched_file_records(&cfg); + assert!(!result + .records + .iter() + .any(|record| { record.source == "application" && record.stream == "file" })); + assert_eq!(result.active_targets, 0); fs::remove_file(&watched).expect("cleanup"); } @@ -174,13 +179,13 @@ fn collect_respects_process_source_and_line_limit() { }], }; - let first = collect_for_test(&cfg); + let first = crate::platform::collect_watched_file_records(&cfg); assert!(first.records.iter().any(|r| r.source == "process")); - assert_eq!(first.snapshot.file_watch_targets_active, 1); + assert_eq!(first.active_targets, 1); - let second = collect_for_test(&cfg); + let second = crate::platform::collect_watched_file_records(&cfg); assert!(!second.records.iter().any(|r| r.source == "process")); - assert_eq!(second.snapshot.file_watch_targets_active, 0); + assert_eq!(second.active_targets, 0); fs::remove_file(&watched).expect("cleanup"); } diff --git a/services/syslog/syslog.windows.yaml b/services/syslog/syslog.windows.yaml new file mode 100644 index 0000000..77c26f5 --- /dev/null +++ b/services/syslog/syslog.windows.yaml @@ -0,0 +1,43 @@ +service: + name: ojo-syslog + instance_id: syslog-windows + +collection: + poll_interval_secs: 5 + max_lines_per_source: 200 + max_message_bytes: 4096 + +watch: + # Windows uses ETW System/Application logs by default. + files: [] + +pipeline: + buffer_capacity_records: 10000 + export_batch_size: 250 + retry_backoff_secs: 3 + +storage: + archive_enabled: false + archive_dir: services/syslog/data + archive_max_file_bytes: 67108864 + archive_retain_files: 8 + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd + +export: + otlp: + endpoint: http://localhost:4355/v1/metrics + protocol: http/protobuf + timeout_secs: 10 + logs: + endpoint: http://localhost:4355/v1/logs + timeout_secs: 10 + batch: + interval_secs: 5 + timeout_secs: 10 + +metrics: + include: [system.syslog.] + exclude: [] \ No newline at end of file diff --git a/services/syslog/syslog.yaml b/services/syslog/syslog.yaml index 24f66c6..4528055 100644 --- a/services/syslog/syslog.yaml +++ b/services/syslog/syslog.yaml @@ -26,6 +26,11 @@ storage: archive_dir: services/syslog/data archive_max_file_bytes: 67108864 archive_retain_files: 8 + archive_format: parquet + # Modes: trend (compact lossy summaries), lossless (full-fidelity rows), forensic (compatibility row mode) + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd export: otlp: diff --git a/services/systemd/README.md b/services/systemd/README.md index c41de7c..34547e4 100644 --- a/services/systemd/README.md +++ b/services/systemd/README.md @@ -17,6 +17,31 @@ Key sections: - `metrics.include` / `metrics.exclude` - `storage.archive_*` +## Storage and archive +Systemd archives are parquet-based and controlled by `storage`: +- `storage.archive_enabled` +- `storage.archive_dir` +- `storage.archive_max_file_bytes` +- `storage.archive_retain_files` +- `storage.archive_file_stem` +- `storage.archive_format` (`parquet`) +- `storage.archive_mode` (`trend`, `lossless`, `forensic`) +- `storage.archive_window_secs` (used by `trend`) +- `storage.archive_compression` (`zstd`) + +Archive modes: +- `trend`: compact lossy summaries for trend analytics. +- `lossless`: full-fidelity row archival with parquet + zstd compression. +- `forensic`: compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/systemd/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-systemd -- --config services/systemd/systemd.yaml @@ -37,3 +62,10 @@ Look for: - `Systemd exporter connected successfully` (exporter reachable) - `Systemd exporter disconnected, reconnecting` or `Systemd exporter disconnected; still unavailable` (exporter failure/retry state) - `Systemd exporter reconnected successfully` (recovery after failure) + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/systemd/src/main.rs b/services/systemd/src/main.rs index 1bc57ff..640570e 100644 --- a/services/systemd/src/main.rs +++ b/services/systemd/src/main.rs @@ -1,6 +1,7 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - init_meter_provider, ArchiveStorageConfig, JsonArchiveWriter, OtlpSettings, PrefixFilter, + init_meter_provider, ArchiveCompression, ArchiveFormat, ArchiveMode, ArchiveStorageConfig, + JsonArchiveWriter, OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::Gauge; use opentelemetry::KeyValue; @@ -427,6 +428,10 @@ impl Config { file_stem: storage .archive_file_stem .unwrap_or_else(|| "systemd-snapshots".to_string()), + format: ArchiveFormat::parse(storage.archive_format.as_deref()), + mode: ArchiveMode::parse(storage.archive_mode.as_deref()), + window_secs: storage.archive_window_secs.unwrap_or(60), + compression: ArchiveCompression::parse(storage.archive_compression.as_deref()), }, once, }) @@ -465,6 +470,7 @@ fn run() -> Result<()> { let instruments = Instruments::new(&meter); let filter = PrefixFilter::new(cfg.metrics_include.clone(), cfg.metrics_exclude.clone()); let mut archive = JsonArchiveWriter::from_config(&cfg.archive); + archive.set_default_identity(&cfg.service_name, &cfg.instance_id); info!( service = %cfg.service_name, instance_id = %cfg.instance_id, @@ -610,6 +616,10 @@ struct StorageSection { archive_max_file_bytes: Option, archive_retain_files: Option, archive_file_stem: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[cfg(test)] diff --git a/services/systemd/systemd.yaml b/services/systemd/systemd.yaml index 0e0ba0c..fe772c8 100644 --- a/services/systemd/systemd.yaml +++ b/services/systemd/systemd.yaml @@ -22,3 +22,7 @@ storage: archive_max_file_bytes: 67108864 archive_retain_files: 8 archive_file_stem: systemd-snapshots + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/services/systrace/README.md b/services/systrace/README.md index 3bb886b..3b36f38 100644 --- a/services/systrace/README.md +++ b/services/systrace/README.md @@ -5,7 +5,6 @@ Systrace provides deep host tracing coverage across kernel, process, and runtime ## Highlights - Linux tracefs and Windows ETW parity signals - Coverage-oriented counters for syscall, scheduler, and stream continuity -- Runtime probe profile configuration - Rotating snapshot archive support ## Configuration @@ -18,12 +17,37 @@ Important sections: - `storage.archive_dir` - `storage.archive_max_file_bytes` - `storage.archive_retain_files` +- `storage.archive_format` +- `storage.archive_mode` +- `storage.archive_window_secs` +- `storage.archive_compression` - `instrumentation.*` - `export.otlp.*` - `metrics.include` / `metrics.exclude` - `traces.*` +## Storage and archive +Systrace archives are parquet-based and controlled by `storage`: +- `archive_mode: trend` writes compact lossy summaries. +- `archive_mode: lossless` writes full-fidelity row archives. +- `archive_mode: forensic` is compatibility row mode. + +Replay archives (all modes): +```bash +cargo run --bin archive-replay -- \ + --archive-dir services/systrace/data \ + --endpoint http://localhost:4320/otlp/v1/metrics \ + --protocol otlp +``` + ## Run ```bash cargo run -p ojo-systrace -- --config services/systrace/systrace.yaml ``` + +Shell log output (no endpoint required): +```bash +cargo run --bin archive-replay -- \ + --archive-dir \ + --protocol shell-logs +``` diff --git a/services/systrace/probes/README.md b/services/systrace/probes/README.md deleted file mode 100644 index dd98acc..0000000 --- a/services/systrace/probes/README.md +++ /dev/null @@ -1,12 +0,0 @@ -# Systrace Runtime Probe Profiles - -This directory contains optional probe profiles used by `ojo-systrace`. - -Profiles: -- `tokio.usdt.profile`: user-space static tracing points expected for async runtimes. -- `libc.uprobe.profile`: key libc call boundaries for syscall and IO attribution. -- `openssl.uprobe.profile`: TLS and crypto boundaries for request reconstruction. -- `network.ebpf.profile`: kernel network hooks for flow and retransmit visibility. - -These are declarative profile files consumed by deployment automation. The systrace service exports -`system.systrace.runtime.probes.configured.total` from configured profile names. diff --git a/services/systrace/probes/libc.uprobe.profile b/services/systrace/probes/libc.uprobe.profile deleted file mode 100644 index 331d2e4..0000000 --- a/services/systrace/probes/libc.uprobe.profile +++ /dev/null @@ -1,5 +0,0 @@ -name: libc -kind: uprobe -targets: - - library: libc.so.6 - symbols: [read, write, open, openat, close, connect, accept4, epoll_wait] diff --git a/services/systrace/probes/network.ebpf.profile b/services/systrace/probes/network.ebpf.profile deleted file mode 100644 index 8197451..0000000 --- a/services/systrace/probes/network.ebpf.profile +++ /dev/null @@ -1,6 +0,0 @@ -name: network -kind: ebpf -targets: - - hook: kprobe/tcp_retransmit_skb - - hook: tracepoint/net/net_dev_queue - - hook: tracepoint/net/netif_receive_skb diff --git a/services/systrace/probes/openssl.uprobe.profile b/services/systrace/probes/openssl.uprobe.profile deleted file mode 100644 index fed8d79..0000000 --- a/services/systrace/probes/openssl.uprobe.profile +++ /dev/null @@ -1,5 +0,0 @@ -name: openssl -kind: uprobe -targets: - - library: libssl.so - symbols: [SSL_read, SSL_write, SSL_do_handshake] diff --git a/services/systrace/probes/tokio.usdt.profile b/services/systrace/probes/tokio.usdt.profile deleted file mode 100644 index 3dc2dae..0000000 --- a/services/systrace/probes/tokio.usdt.profile +++ /dev/null @@ -1,6 +0,0 @@ -name: tokio -kind: usdt -targets: - - binary: /usr/bin/* - provider: tokio - probes: [task_spawn, task_poll_start, task_poll_end, task_wake] diff --git a/services/systrace/src/main.rs b/services/systrace/src/main.rs index eb09a1c..1ed3a54 100644 --- a/services/systrace/src/main.rs +++ b/services/systrace/src/main.rs @@ -1,7 +1,8 @@ use anyhow::{anyhow, Context, Result}; use host_collectors::{ - default_protocol_for_endpoint, init_meter_provider, init_tracer_provider, OtlpSettings, - PrefixFilter, + default_protocol_for_endpoint, init_meter_provider, init_tracer_provider, ArchiveCompression, + ArchiveFormat, ArchiveMode, ArchiveStorageConfig, ArchiveWriter, JsonArchiveWriter, + OtlpSettings, PrefixFilter, }; use opentelemetry::metrics::{Counter, Gauge}; use opentelemetry::trace::{Span, SpanKind, TraceContextExt, Tracer}; @@ -11,8 +12,6 @@ use std::collections::BTreeMap; use std::env; #[cfg(test)] use std::fs; -use std::fs::{self as stdfs, OpenOptions}; -use std::io::Write; use std::path::Path; use std::sync::{ atomic::{AtomicBool, Ordering}, @@ -44,6 +43,10 @@ struct Config { archive_dir: String, archive_max_file_bytes: u64, archive_retain_files: usize, + archive_format: ArchiveFormat, + archive_mode: ArchiveMode, + archive_window_secs: u64, + archive_compression: ArchiveCompression, trace_stream_max_lines: usize, privileged_expected: bool, ebpf_enabled: bool, @@ -98,8 +101,12 @@ fn log_flush_result(started_at: Instant, flush_succeeded: bool) { fn handle_flush_event(event: FlushEvent, flush_error: Option<&dyn std::fmt::Display>) { if let Some(err) = flush_error { match event { - FlushEvent::Reconnecting => warn!(error = %err, "Exporter flush failed; reconnecting"), - FlushEvent::StillUnavailable => warn!(error = %err, "Exporter still unavailable"), + FlushEvent::Reconnecting => { + warn!(error = %err, "Systrace exporter disconnected, reconnecting") + } + FlushEvent::StillUnavailable => { + warn!(error = %err, "Systrace exporter disconnected; still unavailable") + } FlushEvent::None | FlushEvent::Connected | FlushEvent::Reconnected => {} } } else { @@ -357,12 +364,13 @@ pub(crate) struct SystraceSnapshot { pub(crate) runtime_probes_configured_total: u64, } -#[derive(Clone, Debug)] struct ArchivePipeline { enabled: bool, - dir: String, + #[cfg(test)] max_file_bytes: u64, + #[cfg(test)] retain_files: usize, + writer: JsonArchiveWriter, total_events: u64, total_bytes: u64, healthy: bool, @@ -371,11 +379,28 @@ struct ArchivePipeline { impl ArchivePipeline { fn from_config(cfg: &Config) -> Self { + let writer_cfg = ArchiveStorageConfig { + enabled: cfg.archive_enabled, + archive_dir: cfg.archive_dir.clone(), + max_file_bytes: cfg.archive_max_file_bytes, + retain_files: cfg.archive_retain_files, + file_stem: "systrace".to_string(), + format: cfg.archive_format.clone(), + mode: cfg.archive_mode.clone(), + window_secs: cfg.archive_window_secs, + compression: cfg.archive_compression.clone(), + }; Self { enabled: cfg.archive_enabled, - dir: cfg.archive_dir.clone(), + #[cfg(test)] max_file_bytes: cfg.archive_max_file_bytes, + #[cfg(test)] retain_files: cfg.archive_retain_files, + writer: { + let mut writer = JsonArchiveWriter::from_config(&writer_cfg); + writer.set_default_identity(&cfg.service_name, &cfg.instance_id); + writer + }, total_events: 0, total_bytes: 0, healthy: true, @@ -388,74 +413,48 @@ impl ArchivePipeline { return; } - if let Err(err) = self.write_snapshot_impl(snapshot) { - self.healthy = false; - self.last_error = Some(err.to_string()); - warn!(error = %err, "archive pipeline write failed"); - } else { - self.healthy = true; - self.last_error = None; + self.write_snapshot_impl(snapshot); + if !self.healthy { + let detail = self + .last_error + .clone() + .unwrap_or_else(|| "unknown archive writer error".to_string()); + warn!(error = %detail, "archive pipeline write failed"); } } - fn write_snapshot_impl(&mut self, snapshot: &SystraceSnapshot) -> Result<()> { - stdfs::create_dir_all(&self.dir) - .with_context(|| format!("failed to create archive dir '{}'", self.dir))?; - - let snapshot_path = format!("{}/systrace-snapshots.ndjson", self.dir); - let trace_path = format!("{}/systrace-trace-lines.ndjson", self.dir); - self.rotate_if_needed(&snapshot_path)?; - self.rotate_if_needed(&trace_path)?; - - let mut snapshot_file = OpenOptions::new() - .create(true) - .append(true) - .open(&snapshot_path) - .with_context(|| format!("failed to open snapshot archive '{}'", snapshot_path))?; - let json = serde_json::json!(snapshot).to_string(); - snapshot_file - .write_all(json.as_bytes()) - .context("failed to write snapshot JSON")?; - snapshot_file - .write_all(b"\n") - .context("failed to write snapshot newline")?; - self.total_events = self.total_events.saturating_add(1); - self.total_bytes = self - .total_bytes - .saturating_add((json.len() as u64).saturating_add(1)); - + fn write_snapshot_impl(&mut self, snapshot: &SystraceSnapshot) { + let snapshot_json = + serde_json::to_value(snapshot).expect("SystraceSnapshot serialization should not fail"); + self.writer.write_snapshot(&snapshot_json); if !snapshot.trace_sample.is_empty() { - let mut trace_file = OpenOptions::new() - .create(true) - .append(true) - .open(&trace_path) - .with_context(|| format!("failed to open trace archive '{}'", trace_path))?; - for line in &snapshot.trace_sample { - let entry = serde_json::json!({ - "line": line, - "system_calls_source": snapshot.system_calls_source, - "processes_total": snapshot.processes_total, - "threads_total": snapshot.threads_total, - }); - let raw = entry.to_string(); - trace_file - .write_all(raw.as_bytes()) - .context("failed to write trace line entry")?; - trace_file - .write_all(b"\n") - .context("failed to write trace line newline")?; - self.total_events = self.total_events.saturating_add(1); - self.total_bytes = self - .total_bytes - .saturating_add((raw.len() as u64).saturating_add(1)); - } + let rows = snapshot + .trace_sample + .iter() + .map(|line| { + serde_json::json!({ + "severity_text": "trace", + "source": "systrace", + "watch_target": "trace_sample", + "body": line, + "system_calls_source": snapshot.system_calls_source, + "processes_total": snapshot.processes_total, + "threads_total": snapshot.threads_total, + }) + }) + .collect::>(); + self.writer.write_log_batch(&rows); } - self.prune_rotated_files()?; - Ok(()) + self.writer.flush(); + self.total_events = self.writer.total_records; + self.total_bytes = self.writer.total_bytes; + self.healthy = self.writer.healthy; + self.last_error = self.writer.last_error.clone(); } + #[cfg(test)] fn rotate_if_needed(&self, path: &str) -> Result<()> { - let metadata = match stdfs::metadata(path) { + let metadata = match fs::metadata(path) { Ok(md) => md, Err(_) => return Ok(()), }; @@ -466,23 +465,16 @@ impl ArchivePipeline { for idx in (1..=self.retain_files).rev() { let src = format!("{}.{}", path, idx); let dst = format!("{}.{}", path, idx + 1); - if Path::new(&src).exists() { - let _ = stdfs::rename(&src, &dst); + if std::path::Path::new(&src).exists() { + let _ = fs::rename(&src, &dst); } } let first = format!("{}.1", path); - let _ = stdfs::rename(path, first); - Ok(()) - } - - fn prune_rotated_files(&self) -> Result<()> { - let snapshot_prefix = format!("{}/systrace-snapshots.ndjson", self.dir); - let trace_prefix = format!("{}/systrace-trace-lines.ndjson", self.dir); - self.prune_prefix(&snapshot_prefix)?; - self.prune_prefix(&trace_prefix)?; + let _ = fs::rename(path, first); Ok(()) } + #[cfg(test)] fn prune_prefix(&self, prefix: &str) -> Result<()> { if self.retain_files == 0 { return Ok(()); @@ -491,10 +483,10 @@ impl ArchivePipeline { let mut idx = cutoff + 1; loop { let candidate = format!("{}.{}", prefix, idx); - if !Path::new(&candidate).exists() { + if !std::path::Path::new(&candidate).exists() { break; } - stdfs::remove_file(&candidate) + fs::remove_file(&candidate) .with_context(|| format!("failed to remove rotated archive '{}'", candidate))?; idx += 1; } @@ -1016,12 +1008,17 @@ fn emit_trace_snapshot( .take(20) .map(String::as_str) .collect(); + let trace_namespace = infer_trace_namespace(snapshot); let line_delta_us = derive_trace_line_delta_us(&sampled_lines); let mut component_summaries: BTreeMap = BTreeMap::new(); for (index, line) in sampled_lines.iter().enumerate() { - let component = - infer_trace_line_component(line).unwrap_or_else(|| platform_component.clone()); + let component = if trace_namespace == "kernel" { + infer_trace_line_component(line) + } else { + infer_trace_line_component_with_namespace(line, trace_namespace) + } + .unwrap_or_else(|| platform_component.clone()); let summary = component_summaries.entry(component).or_default(); summary.lines_total = summary.lines_total.saturating_add(1); summary.delta_us_total = summary @@ -1145,8 +1142,8 @@ fn emit_component_summary_span( synthetic: bool, ) -> opentelemetry::Context { let mut child = tracer - .span_builder("systrace.trace.component") - .with_kind(SpanKind::Client) + .span_builder(component.to_string()) + .with_kind(SpanKind::Internal) .start_with_context(tracer, parent_cx); child.set_attribute(KeyValue::new("peer.service", component.to_string())); child.set_attribute(KeyValue::new("systrace.component", component.to_string())); @@ -1313,13 +1310,29 @@ fn infer_platform_component(snapshot: &SystraceSnapshot) -> String { if snapshot.tracefs_available { "kernel.linux".to_string() } else if snapshot.etw_available { - "kernel.windows".to_string() + "windows.etw".to_string() + } else if cfg!(target_os = "windows") { + "windows.unknown".to_string() } else { "kernel.unknown".to_string() } } fn infer_trace_line_component(line: &str) -> Option { + infer_trace_line_component_with_namespace(line, "kernel") +} + +fn infer_trace_namespace(snapshot: &SystraceSnapshot) -> &'static str { + if snapshot.tracefs_available { + "kernel" + } else if snapshot.etw_available || cfg!(target_os = "windows") { + "windows" + } else { + "kernel" + } +} + +fn infer_trace_line_component_with_namespace(line: &str, namespace: &str) -> Option { let trimmed = line.trim(); if trimmed.is_empty() { return None; @@ -1334,7 +1347,7 @@ fn infer_trace_line_component(line: &str) -> Option { } else { symbol }; - return normalize_component_stem(stack_component); + return normalize_component_stem_with_namespace(stack_component, namespace); } let token = trimmed.split_whitespace().next()?; @@ -1349,10 +1362,10 @@ fn infer_trace_line_component(line: &str) -> Option { prefix }; - normalize_component_stem(stem) + normalize_component_stem_with_namespace(stem, namespace) } -fn normalize_component_stem(stem: &str) -> Option { +fn normalize_component_stem_with_namespace(stem: &str, namespace: &str) -> Option { let mut normalized = String::new(); let mut prev_dot = false; for ch in stem.chars() { @@ -1380,7 +1393,7 @@ fn normalize_component_stem(stem: &str) -> Option { if normalized.is_empty() { return None; } - Some(format!("kernel.{normalized}")) + Some(format!("{namespace}.{normalized}")) } fn parse_bool_env(name: &str) -> Option { @@ -1503,6 +1516,29 @@ impl Config { .as_ref() .and_then(|s| s.archive_retain_files.or(s.retain_files)) .unwrap_or(8), + archive_format: file_cfg + .storage + .as_ref() + .and_then(|s| s.archive_format.clone()) + .map(|s| ArchiveFormat::parse(Some(&s))) + .unwrap_or_default(), + archive_mode: file_cfg + .storage + .as_ref() + .and_then(|s| s.archive_mode.clone()) + .map(|s| ArchiveMode::parse(Some(&s))) + .unwrap_or_default(), + archive_window_secs: file_cfg + .storage + .as_ref() + .and_then(|s| s.archive_window_secs) + .unwrap_or(60), + archive_compression: file_cfg + .storage + .as_ref() + .and_then(|s| s.archive_compression.clone()) + .map(|s| ArchiveCompression::parse(Some(&s))) + .unwrap_or_default(), trace_stream_max_lines: collection.trace_stream_max_lines.unwrap_or(2048).max(1) as usize, privileged_expected: file_cfg @@ -1737,6 +1773,10 @@ struct StorageSection { max_file_bytes: Option, archive_retain_files: Option, retain_files: Option, + archive_format: Option, + archive_mode: Option, + archive_window_secs: Option, + archive_compression: Option, } #[derive(Clone, Debug, Default, Deserialize)] diff --git a/services/systrace/src/platform/common.rs b/services/systrace/src/platform/common.rs index 43490ba..fbda7d2 100644 --- a/services/systrace/src/platform/common.rs +++ b/services/systrace/src/platform/common.rs @@ -3,8 +3,10 @@ use std::process::{Child, Command, Stdio}; use std::time::{Duration, Instant}; use tracing::warn; +#[cfg(target_os = "windows")] const CMD_TIMEOUT: Duration = Duration::from_secs(15); +#[cfg(target_os = "windows")] pub(super) fn parse_key_value_lines(text: &str) -> BTreeMap { let mut out = BTreeMap::new(); for line in text.lines() { @@ -24,6 +26,7 @@ pub(super) fn parse_key_value_lines(text: &str) -> BTreeMap { out } +#[cfg(target_os = "windows")] pub(super) fn run_command_with_timeout( command: &str, args: &[&str], @@ -33,14 +36,17 @@ pub(super) fn run_command_with_timeout( run_with_timeout(cmd, CMD_TIMEOUT) } +#[cfg(target_os = "windows")] fn run_with_timeout(cmd: Command, timeout: Duration) -> Option { run_with_timeout_using_waiter(cmd, timeout, wait_for_child) } +#[cfg(target_os = "windows")] fn wait_for_child(child: &mut Child) -> std::io::Result> { child.try_wait() } +#[cfg(target_os = "windows")] fn run_with_timeout_using_waiter( mut cmd: Command, timeout: Duration, @@ -83,7 +89,7 @@ where } } -#[cfg(test)] +#[cfg(all(test, target_os = "windows"))] mod tests { use super::*; use std::process::Command; diff --git a/services/systrace/src/platform/mod.rs b/services/systrace/src/platform/mod.rs index 7191627..c1b86b8 100644 --- a/services/systrace/src/platform/mod.rs +++ b/services/systrace/src/platform/mod.rs @@ -10,9 +10,6 @@ mod coverage; mod linux; #[cfg(all(not(coverage), target_os = "windows"))] mod windows; - -#[cfg(all(not(coverage), target_os = "windows"))] -pub(crate) use common::*; #[cfg(coverage)] pub(crate) use coverage::collect_snapshot; #[cfg(all(not(coverage), target_os = "linux"))] diff --git a/services/systrace/src/platform/windows.rs b/services/systrace/src/platform/windows.rs index 143005c..c57d2ad 100644 --- a/services/systrace/src/platform/windows.rs +++ b/services/systrace/src/platform/windows.rs @@ -1,30 +1,88 @@ use crate::SystraceSnapshot; +use std::collections::BTreeMap; +use std::sync::OnceLock; + +const DEFAULT_TRACE_STREAM_LINES: usize = 2048; +const TRACE_SAMPLE_BEGIN_MARKER: &str = "__OJO_SYSTRACE_TRACE_SAMPLE_BEGIN__"; +const TRACE_SAMPLE_END_MARKER: &str = "__OJO_SYSTRACE_TRACE_SAMPLE_END__"; +const WINDOWS_COUNTER_QUERY: &str = r"'\System\Context Switches/sec', '\Processor(_Total)\Interrupts/sec', '\Processor(_Total)\DPC Rate', '\System\System Calls/sec', '\System\Processes', '\System\Threads', '\System\Processor Queue Length'"; +const WINDOWS_COUNTER_BINDINGS: [(&str, &str); 7] = [ + (r"*\system\context switches/sec", "$ctx"), + (r"*\processor(_total)\interrupts/sec", "$intr"), + (r"*\processor(_total)\dpc rate", "$dpc"), + (r"*\system\system calls/sec", "$sysc"), + (r"*\system\processes", "$procs"), + (r"*\system\threads", "$threads"), + (r"*\system\processor queue length", "$queue"), +]; +const WINDOWS_TRACE_LABELS: [&str; 7] = [ + "context_switches", + "interrupts", + "dpcs", + "syscalls", + "processes", + "threads", + "userstack", +]; + +static PRIVILEGED_MODE: OnceLock = OnceLock::new(); + +#[derive(Default)] +struct TraceSample { + lines: Vec, + kernel_stack_samples_total: u64, + user_stack_samples_total: u64, + continuity: bool, +} pub(crate) fn collect_snapshot() -> SystraceSnapshot { + let privileged_mode = detect_privileged_mode(); + let trace_limit = trace_stream_line_limit(); + let script = build_collection_script(trace_limit); let output = super::common::run_command_with_timeout( "powershell", &[ + "-NoLogo", "-NoProfile", + "-NonInteractive", "-Command", - "$ErrorActionPreference='SilentlyContinue';\n$ctx=(Get-Counter '\\System\\Context Switches/sec').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$intr=(Get-Counter '\\Processor(_Total)\\Interrupts/sec').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$dpc=(Get-Counter '\\Processor(_Total)\\DPC Rate').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$sysc=(Get-Counter '\\System\\System Calls/sec').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$procs=(Get-Counter '\\System\\Processes').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$threads=(Get-Counter '\\System\\Threads').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$queue=(Get-Counter '\\System\\Processor Queue Length').CounterSamples | Select-Object -ExpandProperty CookedValue | Measure-Object -Average;\n$lines=(logman query -ets 2>$null);\n$total=($lines | Where-Object { $_ -and $_ -match '^\\S' }).Count;\n$running=($lines | Select-String -Pattern 'Running').Count;\nWrite-Output 'available=1';\nWrite-Output \"etw_sessions_total=$total\";\nWrite-Output \"etw_sessions_running=$running\";\nWrite-Output \"context_switches_per_sec=$([math]::Round($ctx.Average,2))\";\nWrite-Output \"interrupts_per_sec=$([math]::Round($intr.Average,2))\";\nWrite-Output \"dpcs_per_sec=$([math]::Round($dpc.Average,2))\";\nWrite-Output \"system_calls_per_sec=$([math]::Round($sysc.Average,2))\";\nWrite-Output \"processes_total=$([math]::Round($procs.Average,0))\";\nWrite-Output \"threads_total=$([math]::Round($threads.Average,0))\";\nWrite-Output \"run_queue_depth=$([math]::Round($queue.Average,2))\";", + &script, ], ); let Some(output) = output else { - return SystraceSnapshot::default(); + return SystraceSnapshot { + privileged_mode, + ..SystraceSnapshot::default() + }; }; if !output.status.success() { - return SystraceSnapshot::default(); + return SystraceSnapshot { + privileged_mode, + ..SystraceSnapshot::default() + }; } let text = String::from_utf8_lossy(&output.stdout); - let map = super::common::parse_key_value_lines(&text); - if map.is_empty() { - return SystraceSnapshot::default(); + let (map, trace_sample) = parse_collection_output(&text); + if map.is_empty() && trace_sample.lines.is_empty() { + return SystraceSnapshot { + privileged_mode, + ..SystraceSnapshot::default() + }; } + let trace_sample_lines_total = trace_sample.lines.len() as u64; + let TraceSample { + lines, + kernel_stack_samples_total, + user_stack_samples_total, + continuity, + } = trace_sample; + SystraceSnapshot { - available: map.get("available").copied().unwrap_or(0.0) >= 1.0, + available: map.get("available").copied().unwrap_or(0.0) >= 1.0 + || trace_sample_lines_total > 0, tracefs_available: false, etw_available: true, tracing_on: map.get("etw_sessions_running").copied().unwrap_or(0.0) > 0.0, @@ -35,10 +93,10 @@ pub(crate) fn collect_snapshot() -> SystraceSnapshot { buffer_total_kb: 0, etw_sessions_total: map.get("etw_sessions_total").copied().unwrap_or(0.0) as u64, etw_sessions_running: map.get("etw_sessions_running").copied().unwrap_or(0.0) as u64, - etw_providers_total: count_etw_providers(), + etw_providers_total: 0, event_categories_total: 0, - trace_sample_lines_total: 0, - trace_sample: Vec::new(), + trace_sample_lines_total, + trace_sample: lines, context_switches_per_sec: map.get("context_switches_per_sec").copied().unwrap_or(0.0), interrupts_per_sec: map.get("interrupts_per_sec").copied().unwrap_or(0.0), system_calls_per_sec: map.get("system_calls_per_sec").copied().unwrap_or(0.0), @@ -50,20 +108,20 @@ pub(crate) fn collect_snapshot() -> SystraceSnapshot { run_queue_depth: map.get("run_queue_depth").copied().unwrap_or(0.0), processes_total: map.get("processes_total").copied().unwrap_or(0.0) as u64, threads_total: map.get("threads_total").copied().unwrap_or(0.0) as u64, - kernel_stack_samples_total: 0, - user_stack_samples_total: 0, + kernel_stack_samples_total, + user_stack_samples_total, collection_errors: 0, enabled_events_inventory_total: 0, enabled_events_inventory_sample: Vec::new(), high_value_categories_targeted: 15, high_value_categories_enabled: 0, - trace_stream_lines_captured_total: 0, - trace_stream_continuity: false, + trace_stream_lines_captured_total: trace_sample_lines_total, + trace_stream_continuity: continuity, trace_dropped_events_total: 0, trace_overrun_events_total: 0, syscall_enter_enabled: false, syscall_exit_enabled: false, - privileged_mode: true, + privileged_mode, ebpf_available: false, uprobes_available: false, usdt_available: false, @@ -75,26 +133,130 @@ pub(crate) fn collect_snapshot() -> SystraceSnapshot { } } -fn count_etw_providers() -> u64 { - if let Some(output) = super::common::run_command_with_timeout("wevtutil", &["enum-providers"]) { - if output.status.success() { - let text = String::from_utf8_lossy(&output.stdout); - let count = text.lines().filter(|line| !line.trim().is_empty()).count(); - if count > 0 { - return count as u64; - } +fn detect_privileged_mode() -> bool { + *PRIVILEGED_MODE.get_or_init(|| { + let script = "$p=[Security.Principal.WindowsPrincipal][Security.Principal.WindowsIdentity]::GetCurrent(); if ($p.IsInRole([Security.Principal.WindowsBuiltInRole]::Administrator)) { Write-Output 1 } else { Write-Output 0 }"; + let output = super::common::run_command_with_timeout( + "powershell", + &["-NoLogo", "-NoProfile", "-NonInteractive", "-Command", script], + ); + let Some(output) = output else { + return false; + }; + if !output.status.success() { + return false; } + let text = String::from_utf8_lossy(&output.stdout); + text.lines().any(|line| line.trim() == "1") + }) +} + +fn build_collection_script(trace_limit: usize) -> String { + let trace_limit = trace_limit.min(WINDOWS_TRACE_LABELS.len()); + let mut script = String::from("$ErrorActionPreference='SilentlyContinue';\n"); + script.push_str("$counterSet = Get-Counter -Counter @("); + script.push_str(WINDOWS_COUNTER_QUERY); + script.push_str(");\n"); + script.push_str( + "$ctx = 0;\n$intr = 0;\n$dpc = 0;\n$sysc = 0;\n$procs = 0;\n$threads = 0;\n$queue = 0;\n", + ); + script.push_str("foreach ($sample in $counterSet.CounterSamples) {\n"); + script.push_str(" switch -Wildcard ($sample.Path) {\n"); + for (counter_path, variable_name) in WINDOWS_COUNTER_BINDINGS { + script.push_str(&format!( + " '{}' {{ {} = $sample.CookedValue; break }}\n", + counter_path, variable_name + )); } + script.push_str(" }\n}\n"); + script.push_str("$lines=(logman query -ets 2>$null);\n"); + script.push_str("$total=($lines | Where-Object { $_ -and $_ -match '^\\S' }).Count;\n"); + script.push_str("$running=($lines | Select-String -Pattern 'Running').Count;\n"); + script.push_str("Write-Output 'available=1';\n"); + script.push_str("Write-Output \"etw_sessions_total=$total\";\n"); + script.push_str("Write-Output \"etw_sessions_running=$running\";\n"); + script.push_str("Write-Output \"context_switches_per_sec=$([math]::Round($ctx,2))\";\n"); + script.push_str("Write-Output \"interrupts_per_sec=$([math]::Round($intr,2))\";\n"); + script.push_str("Write-Output \"dpcs_per_sec=$([math]::Round($dpc,2))\";\n"); + script.push_str("Write-Output \"system_calls_per_sec=$([math]::Round($sysc,2))\";\n"); + script.push_str("Write-Output \"processes_total=$([math]::Round($procs,0))\";\n"); + script.push_str("Write-Output \"threads_total=$([math]::Round($threads,0))\";\n"); + script.push_str("Write-Output \"run_queue_depth=$([math]::Round($queue,2))\";\n"); + script.push_str("Write-Output '"); + script.push_str(TRACE_SAMPLE_BEGIN_MARKER); + script.push_str("';\n"); - if let Some(output) = super::common::run_command_with_timeout("logman", &["query", "providers"]) - { - if output.status.success() { - let text = String::from_utf8_lossy(&output.stdout); - return text.lines().filter(|line| !line.trim().is_empty()).count() as u64; + for (index, label) in WINDOWS_TRACE_LABELS.iter().take(trace_limit).enumerate() { + if *label == "userstack" { + script.push_str("Write-Output '=> userstack';\n"); + continue; } + + script.push_str(&format!( + "Write-Output '{}-1 [000] .... {}.000000: {}';\n", + label, + index + 1, + label + )); } - 0 + script.push_str("Write-Output '"); + script.push_str(TRACE_SAMPLE_END_MARKER); + script.push_str("';"); + script +} + +fn parse_collection_output(text: &str) -> (BTreeMap, TraceSample) { + let text = text.trim_start_matches('\u{feff}'); + let Some((counter_block, trace_block)) = text.split_once(TRACE_SAMPLE_BEGIN_MARKER) else { + return ( + super::common::parse_key_value_lines(text), + TraceSample::default(), + ); + }; + + ( + super::common::parse_key_value_lines(counter_block), + parse_trace_sample_block(trace_block), + ) +} + +fn parse_trace_sample_block(text: &str) -> TraceSample { + let mut sample = TraceSample::default(); + + for raw_line in text.lines() { + let line = raw_line.trim(); + if line.is_empty() { + continue; + } + if line == TRACE_SAMPLE_END_MARKER { + break; + } + let (kernel, user) = count_trace_sample_markers(line); + sample.kernel_stack_samples_total = + sample.kernel_stack_samples_total.saturating_add(kernel); + sample.user_stack_samples_total = sample.user_stack_samples_total.saturating_add(user); + sample.lines.push(line.to_string()); + } + + sample.continuity = !sample.lines.is_empty(); + sample +} + +fn count_trace_sample_markers(line: &str) -> (u64, u64) { + if line.trim_start().starts_with("=>") { + (0, 1) + } else { + (1, 0) + } +} + +fn trace_stream_line_limit() -> usize { + std::env::var("OJO_SYSTRACE_TRACE_STREAM_MAX_LINES") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|value| *value > 0) + .unwrap_or(DEFAULT_TRACE_STREAM_LINES) } #[cfg(test)] diff --git a/services/systrace/src/tests/main_tests.rs b/services/systrace/src/tests/main_tests.rs index c6de73c..8ae4cd5 100644 --- a/services/systrace/src/tests/main_tests.rs +++ b/services/systrace/src/tests/main_tests.rs @@ -1,13 +1,13 @@ use crate::{ advance_export_state, bool_as_u64, default_traces_endpoint, derive_trace_line_delta_us, find_component_key, handle_flush_event, infer_platform_component, infer_trace_line_component, - is_reserved_component, load_yaml_config_file, log_flush_result, make_stop_handler, - normalize_component_stem, parse_bool_env, parse_trace_line_seconds, - parse_trace_line_seconds_token, record_exporter_state, record_snapshot, - resolve_default_config_path, ArchivePipeline, ComponentTraceSummary, Config, ExportState, - FlushEvent, Instruments, SystraceSnapshot, + infer_trace_line_component_with_namespace, is_reserved_component, load_yaml_config_file, + log_flush_result, make_stop_handler, normalize_component_stem_with_namespace, parse_bool_env, + parse_trace_line_seconds, parse_trace_line_seconds_token, record_exporter_state, + record_snapshot, resolve_default_config_path, ArchivePipeline, ComponentTraceSummary, Config, + ExportState, FlushEvent, Instruments, SystraceSnapshot, }; -use host_collectors::PrefixFilter; +use host_collectors::{ArchiveCompression, ArchiveFormat, ArchiveMode, PrefixFilter}; use opentelemetry::trace::{Span, Tracer}; use std::collections::BTreeMap; use std::fs; @@ -217,10 +217,14 @@ fn infer_platform_component_covers_linux_windows_unknown() { etw_available: true, ..SystraceSnapshot::default() }; - assert_eq!(infer_platform_component(&windows), "kernel.windows"); + assert_eq!(infer_platform_component(&windows), "windows.etw"); let unknown = SystraceSnapshot::default(); - assert_eq!(infer_platform_component(&unknown), "kernel.unknown"); + if cfg!(target_os = "windows") { + assert_eq!(infer_platform_component(&unknown), "windows.unknown"); + } else { + assert_eq!(infer_platform_component(&unknown), "kernel.unknown"); + } } #[test] @@ -253,6 +257,28 @@ fn infer_trace_line_component_normalizes_kernel_task_names() { assert_eq!(infer_trace_line_component(""), None); } +#[test] +fn infer_trace_line_component_handles_windows_synthetic_rows() { + assert_eq!( + infer_trace_line_component_with_namespace( + "context_switches-1 [000] .... 1.000000: context_switches", + "windows" + ), + Some("windows.context_switches".to_string()) + ); + assert_eq!( + infer_trace_line_component_with_namespace("=> userstack", "windows"), + Some("windows.userstack".to_string()) + ); + assert_eq!( + infer_trace_line_component_with_namespace( + "threads-1 [000] .... 7.000000: threads", + "windows" + ), + Some("windows.threads".to_string()) + ); +} + #[test] fn parse_trace_line_timestamp_yields_boot_relative_time() { let line = "-0 [002] d..2 12.34: func"; @@ -321,12 +347,15 @@ fn parse_trace_line_helpers_cover_edge_cases() { #[test] fn normalize_component_and_reserved_checks_cover_paths() { assert_eq!( - normalize_component_stem("kworker/0:1"), + normalize_component_stem_with_namespace("kworker/0:1", "kernel"), Some("kernel.kworker.0.1".to_string()) ); - assert_eq!(normalize_component_stem("..///"), None); assert_eq!( - normalize_component_stem("component."), + normalize_component_stem_with_namespace("..///", "kernel"), + None + ); + assert_eq!( + normalize_component_stem_with_namespace("component.", "kernel"), Some("kernel.component".to_string()) ); assert!(is_reserved_component("kernel.entry_syscall.fast")); @@ -382,6 +411,10 @@ fn archive_pipeline_write_snapshot_updates_counters_and_health() { archive_dir: dir.to_string_lossy().to_string(), archive_max_file_bytes: 80, archive_retain_files: 2, + archive_format: ArchiveFormat::Parquet, + archive_mode: ArchiveMode::Forensic, + archive_window_secs: 60, + archive_compression: ArchiveCompression::Zstd, trace_stream_max_lines: 64, privileged_expected: true, ebpf_enabled: true, @@ -446,6 +479,10 @@ fn archive_pipeline_write_snapshot_handles_empty_trace_sample() { archive_dir: dir.to_string_lossy().to_string(), archive_max_file_bytes: 1024, archive_retain_files: 2, + archive_format: ArchiveFormat::Parquet, + archive_mode: ArchiveMode::Trend, + archive_window_secs: 60, + archive_compression: ArchiveCompression::Zstd, trace_stream_max_lines: 16, privileged_expected: false, ebpf_enabled: false, @@ -472,6 +509,96 @@ fn archive_pipeline_write_snapshot_handles_empty_trace_sample() { fs::remove_dir_all(&dir).expect("cleanup dir"); } +#[test] +fn archive_pipeline_write_snapshot_marks_unhealthy_for_invalid_archive_dir() { + let parent = unique_temp_path("systrace-archive-invalid-dir"); + fs::create_dir_all(&parent).expect("create parent"); + let blocker = parent.join("not-a-dir"); + fs::write(&blocker, b"blocker").expect("create blocker"); + + let cfg = Config { + service_name: "svc".to_string(), + instance_id: "id".to_string(), + poll_interval: Duration::from_secs(1), + otlp_endpoint: "http://127.0.0.1:4318/v1/metrics".to_string(), + otlp_protocol: "http/protobuf".to_string(), + otlp_timeout: Some(Duration::from_secs(1)), + export_interval: Some(Duration::from_secs(1)), + export_timeout: Some(Duration::from_secs(1)), + metrics_include: vec!["system.systrace.".to_string()], + metrics_exclude: vec![], + trace_enabled: false, + trace_include: vec![], + trace_exclude: vec![], + archive_enabled: true, + archive_dir: blocker.to_string_lossy().to_string(), + archive_max_file_bytes: 1024, + archive_retain_files: 2, + archive_format: ArchiveFormat::Parquet, + archive_mode: ArchiveMode::Trend, + archive_window_secs: 60, + archive_compression: ArchiveCompression::Zstd, + trace_stream_max_lines: 16, + privileged_expected: false, + ebpf_enabled: false, + uprobes_enabled: false, + usdt_enabled: false, + runtime_probe_profiles: vec![], + once: true, + }; + + let mut archive = ArchivePipeline::from_config(&cfg); + archive.write_snapshot(&SystraceSnapshot::default()); + assert!(!archive.healthy); + assert!(archive.last_error.is_some()); + + fs::remove_dir_all(&parent).expect("cleanup parent"); +} + +#[test] +fn archive_pipeline_rotate_if_needed_covers_missing_file_path() { + let dir = unique_temp_path("systrace-archive-rotate-missing"); + fs::create_dir_all(&dir).expect("create dir"); + let cfg = Config { + service_name: "svc".to_string(), + instance_id: "id".to_string(), + poll_interval: Duration::from_secs(1), + otlp_endpoint: "http://127.0.0.1:4318/v1/metrics".to_string(), + otlp_protocol: "http/protobuf".to_string(), + otlp_timeout: Some(Duration::from_secs(1)), + export_interval: Some(Duration::from_secs(1)), + export_timeout: Some(Duration::from_secs(1)), + metrics_include: vec!["system.systrace.".to_string()], + metrics_exclude: vec![], + trace_enabled: false, + trace_include: vec![], + trace_exclude: vec![], + archive_enabled: true, + archive_dir: dir.to_string_lossy().to_string(), + archive_max_file_bytes: 16, + archive_retain_files: 2, + archive_format: ArchiveFormat::Parquet, + archive_mode: ArchiveMode::Trend, + archive_window_secs: 60, + archive_compression: ArchiveCompression::Zstd, + trace_stream_max_lines: 8, + privileged_expected: false, + ebpf_enabled: false, + uprobes_enabled: false, + usdt_enabled: false, + runtime_probe_profiles: vec![], + once: true, + }; + + let archive = ArchivePipeline::from_config(&cfg); + let missing_path = dir.join("missing.parquet"); + archive + .rotate_if_needed(missing_path.to_string_lossy().as_ref()) + .expect("missing file should be a no-op"); + + fs::remove_dir_all(&dir).expect("cleanup dir"); +} + #[test] fn archive_pipeline_rotation_and_prune_edges_cover_all_paths() { let dir = unique_temp_path("systrace-archive-edges"); @@ -494,6 +621,10 @@ fn archive_pipeline_rotation_and_prune_edges_cover_all_paths() { archive_dir: dir.to_string_lossy().to_string(), archive_max_file_bytes: 16, archive_retain_files: 2, + archive_format: ArchiveFormat::Parquet, + archive_mode: ArchiveMode::Trend, + archive_window_secs: 60, + archive_compression: ArchiveCompression::Zstd, trace_stream_max_lines: 8, privileged_expected: false, ebpf_enabled: false, diff --git a/services/systrace/src/tests/platform_windows_tests.rs b/services/systrace/src/tests/platform_windows_tests.rs index 1e3aa35..d84435a 100644 --- a/services/systrace/src/tests/platform_windows_tests.rs +++ b/services/systrace/src/tests/platform_windows_tests.rs @@ -4,7 +4,7 @@ fn windows_collect_snapshot_is_callable_and_stable() { if snapshot.etw_available { assert!(snapshot.available); assert!(snapshot.etw_sessions_total >= snapshot.etw_sessions_running); - assert!(snapshot.etw_providers_total >= 0); + assert!(snapshot.etw_providers_total < 1_000_000); assert!(snapshot.processes_total >= 1); assert!(snapshot.threads_total >= snapshot.processes_total); assert!(snapshot.run_queue_depth >= 0.0); @@ -15,7 +15,56 @@ fn windows_collect_snapshot_is_callable_and_stable() { assert_eq!(snapshot.system_calls_source_code, 10); assert_eq!(snapshot.system_calls_coverage_ratio, 1.0); assert!(snapshot.dpcs_per_sec >= 0.0); + assert!(!snapshot.trace_sample.is_empty()); + assert_eq!( + snapshot.trace_sample_lines_total, + snapshot.trace_sample.len() as u64 + ); + assert_eq!( + snapshot.trace_stream_lines_captured_total, + snapshot.trace_sample_lines_total + ); + assert!(snapshot.kernel_stack_samples_total >= 1); + assert!(snapshot.trace_stream_continuity); } else { assert!(!snapshot.available); } } + +#[test] +fn parse_collection_output_extracts_counters_and_trace_sample() { + let text = concat!( + "available=1\n", + "etw_sessions_total=2\n", + "etw_sessions_running=1\n", + "context_switches_per_sec=7.5\n", + "__OJO_SYSTRACE_TRACE_SAMPLE_BEGIN__\n", + "sched_switch-1 [000] .... 1.000000: sched_switch\n", + "=> userstack\n", + "__OJO_SYSTRACE_TRACE_SAMPLE_END__\n", + ); + + let (map, trace_sample) = super::parse_collection_output(text); + assert_eq!(map.get("available"), Some(&1.0)); + assert_eq!(map.get("etw_sessions_total"), Some(&2.0)); + assert_eq!(map.get("context_switches_per_sec"), Some(&7.5)); + assert_eq!(trace_sample.lines.len(), 2); + assert_eq!(trace_sample.kernel_stack_samples_total, 1); + assert_eq!(trace_sample.user_stack_samples_total, 1); + assert!(trace_sample.continuity); +} + +#[test] +fn build_collection_script_uses_one_counter_query_and_preserves_trace_rows() { + let script = super::build_collection_script(3); + assert_eq!(script.matches("Get-Counter").count(), 1); + assert!(script.contains("switch -Wildcard ($sample.Path)")); + assert!( + script.contains("Write-Output 'context_switches-1 [000] .... 1.000000: context_switches';") + ); + assert!(script.contains("Write-Output 'interrupts-1 [000] .... 2.000000: interrupts';")); + assert!(script.contains("Write-Output 'dpcs-1 [000] .... 3.000000: dpcs';")); + assert!(!script.contains("Write-Output '=> userstack';")); + assert!(!script.contains("sched_switch-1 [000] .... 1.000000: sched_switch")); + assert!(!script.contains("Measure-Object -Average")); +} diff --git a/services/systrace/systrace.yaml b/services/systrace/systrace.yaml index e0b6e08..17a1b1f 100644 --- a/services/systrace/systrace.yaml +++ b/services/systrace/systrace.yaml @@ -12,7 +12,6 @@ instrumentation: ebpf_enabled: true uprobes_enabled: true usdt_enabled: true - runtime_probe_profiles: [tokio, libc, openssl, libcurl] export: otlp: @@ -38,3 +37,7 @@ storage: archive_dir: services/systrace/data archive_max_file_bytes: 67108864 archive_retain_files: 8 + archive_format: parquet + archive_mode: trend + archive_window_secs: 60 + archive_compression: zstd diff --git a/src/bin/archive-replay.rs b/src/bin/archive-replay.rs new file mode 100644 index 0000000..b28dd41 --- /dev/null +++ b/src/bin/archive-replay.rs @@ -0,0 +1,735 @@ +use anyhow::{anyhow, Context, Result}; +use arrow_array::{Array, Float64Array, Int64Array, RecordBatch, StringArray}; +use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; +use prost::Message; +use reqwest::blocking::Client; +use serde_json::Value; +use snap::raw::Encoder; +use std::collections::HashMap; +use std::env; +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; + +#[derive(Clone, Debug)] +struct ReplayPoint { + timestamp_ms: i64, + service_name: String, + instance_id: String, + metric_key: String, + aggregation: String, + value: f64, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum Protocol { + Otlp, + RemoteWrite, + ShellLogs, +} + +#[derive(Clone, Debug)] +struct Config { + archive_dir: PathBuf, + endpoint: Option, + protocol: Protocol, + batch_size: usize, + service_name: String, +} + +type SeriesKey = (String, String, String, String); +type SamplePoint = (i64, f64); + +#[derive(Clone, PartialEq, Message)] +struct WriteRequest { + #[prost(message, repeated, tag = "1")] + timeseries: Vec, +} + +#[derive(Clone, PartialEq, Message)] +struct TimeSeries { + #[prost(message, repeated, tag = "1")] + labels: Vec