From 3590c547d2fa490b40256dbf3ef41448221e2ab2 Mon Sep 17 00:00:00 2001
From: Mostafa Moradian <mostafa@tigerdata.com>
Date: Sun, 18 Oct 2026 20:52:41 +0200
Subject: [PATCH 1/6] feat(eval): make HIR retention for save_hir opt-in

---
 crates/rsigma-eval/README.md           |  3 ++
 crates/rsigma-eval/src/engine/mod.rs   | 75 ++++++++++++++++++++++----
 crates/rsigma-eval/src/engine/tests.rs | 24 +++++++++
 crates/rsigma-eval/tests/hir_cache.rs  | 39 +++++++++++++-
 docs/content/library/eval.md           |  3 +-
 docs/content/library/ir.md             |  2 +-
 docs/content/reference/architecture.md |  2 +-
 7 files changed, 133 insertions(+), 15 deletions(-)

diff --git a/crates/rsigma-eval/README.md b/crates/rsigma-eval/README.md
index 541a632f3..34b391d4e 100644
--- a/crates/rsigma-eval/README.md
+++ b/crates/rsigma-eval/README.md
@@ -36,6 +36,9 @@ The free function `apply_filters(collection: &SigmaCollection) -> Vec<SigmaRule>
 | `bloom_max_bytes()` | Configured bloom memory budget, or `None` if using the default |
 | `set_cross_rule_ac(enabled: bool)` | Enable opt-in cross-rule Aho-Corasick pre-filter (requires `daachorse-index` feature; see [Cross-Rule AC Index](#cross-rule-aho-corasick-index-opt-in-feature-gated)) |
 | `cross_rule_ac_enabled()` | Whether the cross-rule AC pre-filter is currently enabled (`daachorse-index` only) |
+| `set_retain_hir(enabled: bool)` | Keep each loaded rule's HIR for `save_hir` (off by default; enable before loading rules) |
+| `retain_hir_enabled()` | Whether loaded rules keep their HIR |
+| `save_hir()` / `load_hir(bytes)` | Write the retained HIR as a versioned restart cache, and load rules from one; `save_hir` errors unless retention was on for every rule loaded from parsed Sigma |
 | `rule_count()` | Number of loaded rules |
 | `rules()` | Access the compiled rules slice |
 | `transform_rule(rule: &SigmaRule)` | Apply the engine's pipelines and return the rewritten rule as a `TransformedRule`, without compiling or loading (inspection only) |
diff --git a/crates/rsigma-eval/src/engine/mod.rs b/crates/rsigma-eval/src/engine/mod.rs
index 165cebe6b..b3b00bbde 100644
--- a/crates/rsigma-eval/src/engine/mod.rs
+++ b/crates/rsigma-eval/src/engine/mod.rs
@@ -90,10 +90,16 @@ pub(crate) struct IdentifiedEvaluation {
 pub struct Engine<L = LogSourceExtractor> {
     rules: Vec<CompiledRule>,
     /// Post-pipeline, pre-filter HIR for rules added via the parsed-rule paths,
-    /// retained so [`Engine::save_hir`] can serialize a restart cache. Kept in
-    /// step with `rules` for those paths; rules added via `add_compiled_rule`
-    /// have no HIR and are not represented here.
+    /// retained so [`Engine::save_hir`] can serialize a restart cache. Empty
+    /// unless [`Engine::set_retain_hir`] enabled retention; rules added via
+    /// `add_compiled_rule` have no HIR and are not represented here.
     ir_rules: Vec<IrRule>,
+    /// Whether the parsed-rule paths keep each rule's HIR in `ir_rules`.
+    retain_hir: bool,
+    /// Rules added via the parsed-rule paths while retention was off. A
+    /// non-zero count makes `save_hir` fail instead of writing a cache that
+    /// silently misses them.
+    unretained_hir: usize,
     pipelines: Vec<Pipeline>,
     /// Global override: include the full event JSON in all match results.
     /// When `true`, overrides per-rule `rsigma.include_event` custom attributes.
@@ -186,6 +192,8 @@ impl<L> Engine<L> {
         Engine {
             rules: Vec::new(),
             ir_rules: Vec::new(),
+            retain_hir: false,
+            unretained_hir: 0,
             pipelines,
             include_event: false,
             match_detail: MatchDetailLevel::Off,
@@ -341,6 +349,26 @@ impl<L> Engine<L> {
         self.cross_rule_ac_enabled
     }
 
+    /// Enable or disable retention of each loaded rule's HIR for
+    /// [`Engine::save_hir`].
+    ///
+    /// Off by default: nothing on the evaluation path reads the HIR, so
+    /// keeping it only costs memory. Enable it before loading rules when the
+    /// engine's rules will be saved as a restart cache. Disabling it drops
+    /// any HIR already retained.
+    pub fn set_retain_hir(&mut self, enabled: bool) {
+        if !enabled {
+            self.unretained_hir += self.ir_rules.len();
+            self.ir_rules = Vec::new();
+        }
+        self.retain_hir = enabled;
+    }
+
+    /// Returns whether loaded rules keep their HIR for [`Engine::save_hir`].
+    pub fn retain_hir_enabled(&self) -> bool {
+        self.retain_hir
+    }
+
     /// Set global `include_event` — when `true`, all match results include
     /// the full event JSON regardless of per-rule custom attributes.
     pub fn set_include_event(&mut self, include: bool) {
@@ -431,20 +459,29 @@ impl<L> Engine<L> {
     }
 
     /// Lower a rule to HIR (applying any configured pipelines first), compile
-    /// it, and store both the HIR and the compiled rule. Shared by the single-
-    /// and batched-add paths so they stay behaviourally identical, and so the
-    /// retained HIR (`ir_rules`) tracks `rules` for [`Engine::save_hir`].
+    /// it, and store the compiled rule, plus its HIR when retention is on.
+    /// Shared by the single- and batched-add paths so they stay behaviourally
+    /// identical, and so the retained HIR (`ir_rules`) tracks `rules` for
+    /// [`Engine::save_hir`].
     ///
     /// Both pushes happen only after lowering and compilation succeed, so a
     /// failing rule leaves neither vector mutated.
     fn compile_and_store(&mut self, rule: &SigmaRule) -> Result<()> {
         let ir = self.lower_with_pipelines(rule)?;
         let compiled = compile_to_compiled(&ir)?;
-        self.ir_rules.push(ir);
-        self.rules.push(compiled);
+        self.store(ir, compiled);
         Ok(())
     }
 
+    fn store(&mut self, ir: IrRule, compiled: CompiledRule) {
+        if self.retain_hir {
+            self.ir_rules.push(ir);
+        } else {
+            self.unretained_hir += 1;
+        }
+        self.rules.push(compiled);
+    }
+
     /// Lower a rule to HIR, applying any configured pipelines first.
     fn lower_with_pipelines(&self, rule: &SigmaRule) -> Result<IrRule> {
         if self.pipelines.is_empty() {
@@ -465,7 +502,23 @@ impl<L> Engine<L> {
     /// injections (applied to the compiled rules) or rules added via
     /// [`Engine::add_compiled_rule`] / [`Engine::extend_compiled_rules`].
     /// Re-apply any filters after [`Engine::load_hir`] if the engine used them.
+    ///
+    /// Requires HIR retention ([`Engine::set_retain_hir`]) to have been on
+    /// for every rule loaded through those paths; otherwise this returns
+    /// [`EvalError::HirCache`] rather than a cache that misses rules.
     pub fn save_hir(&self) -> Result<Vec<u8>> {
+        if !self.retain_hir {
+            return Err(EvalError::HirCache(
+                "HIR retention is off; call Engine::set_retain_hir(true) before loading rules"
+                    .to_string(),
+            ));
+        }
+        if self.unretained_hir > 0 {
+            return Err(EvalError::HirCache(format!(
+                "{} rules were loaded while HIR retention was off and cannot be saved",
+                self.unretained_hir
+            )));
+        }
         Ok(rsigma_ir::encode_rules(&self.ir_rules)?)
     }
 
@@ -474,13 +527,13 @@ impl<L> Engine<L> {
     ///
     /// Rules are appended to any already loaded, so a warm start loads into a
     /// fresh [`Engine::new`]. A blob whose schema version differs from this
-    /// build's is rejected (see [`rsigma_ir::decode_rules`]).
+    /// build's is rejected (see [`rsigma_ir::decode_rules`]). The loaded HIR
+    /// is retained only when [`Engine::set_retain_hir`] is on.
     pub fn load_hir(&mut self, bytes: &[u8]) -> Result<()> {
         let rules = rsigma_ir::decode_rules(bytes)?;
         for ir in rules {
             let compiled = compile_to_compiled(&ir)?;
-            self.ir_rules.push(ir);
-            self.rules.push(compiled);
+            self.store(ir, compiled);
         }
         self.rebuild_index();
         Ok(())
diff --git a/crates/rsigma-eval/src/engine/tests.rs b/crates/rsigma-eval/src/engine/tests.rs
index b75607bca..3374dde11 100644
--- a/crates/rsigma-eval/src/engine/tests.rs
+++ b/crates/rsigma-eval/src/engine/tests.rs
@@ -2721,3 +2721,27 @@ detection:
     assert!(matches(&engine, &json!({"tags": []})));
     assert!(matches(&engine, &json!({"other": 1})));
 }
+
+#[test]
+fn hir_is_retained_only_when_enabled() {
+    let yaml = r#"
+title: Detect Whoami
+logsource:
+    product: windows
+detection:
+    selection:
+        CommandLine|contains: 'whoami'
+    condition: selection
+"#;
+    let collection = parse_sigma_yaml(yaml).unwrap();
+
+    let mut engine = Engine::new();
+    engine.add_collection(&collection).unwrap();
+    assert_eq!(engine.rules.len(), 1);
+    assert!(engine.ir_rules.is_empty());
+
+    let mut engine = Engine::new();
+    engine.set_retain_hir(true);
+    engine.add_collection(&collection).unwrap();
+    assert_eq!(engine.ir_rules.len(), 1);
+}
diff --git a/crates/rsigma-eval/tests/hir_cache.rs b/crates/rsigma-eval/tests/hir_cache.rs
index ea22656c1..f95700fec 100644
--- a/crates/rsigma-eval/tests/hir_cache.rs
+++ b/crates/rsigma-eval/tests/hir_cache.rs
@@ -59,6 +59,7 @@ fn save_load_round_trip_matches_original() {
     let collection = parse_sigma_yaml(RULES).expect("parse");
 
     let mut warm = Engine::new();
+    warm.set_retain_hir(true);
     warm.add_collection(&collection).expect("add");
 
     let blob = warm.save_hir().expect("save_hir");
@@ -91,6 +92,7 @@ fn save_load_round_trip_matches_original() {
 fn load_rejects_schema_mismatch() {
     let collection = parse_sigma_yaml(RULES).expect("parse");
     let mut warm = Engine::new();
+    warm.set_retain_hir(true);
     warm.add_collection(&collection).expect("add");
     let mut blob = warm.save_hir().expect("save_hir");
 
@@ -105,10 +107,45 @@ fn load_rejects_schema_mismatch() {
 
 #[test]
 fn empty_engine_round_trips() {
-    let warm = Engine::new();
+    let mut warm = Engine::new();
+    warm.set_retain_hir(true);
     let blob = warm.save_hir().expect("save_hir");
     let mut cold = Engine::new();
     cold.load_hir(&blob).expect("load_hir");
     let out = results(&cold, &json!({"CommandLine": "whoami"}));
     assert!(out.is_empty());
 }
+
+#[test]
+fn save_requires_retention() {
+    let collection = parse_sigma_yaml(RULES).expect("parse");
+    let mut engine = Engine::new();
+    assert!(!engine.retain_hir_enabled());
+    engine.add_collection(&collection).expect("add");
+    let err = engine.save_hir().expect_err("retention is off by default");
+    assert!(err.to_string().contains("set_retain_hir(true)"), "{err}");
+}
+
+#[test]
+fn save_refuses_rules_loaded_before_retention() {
+    let collection = parse_sigma_yaml(RULES).expect("parse");
+    let mut engine = Engine::new();
+    engine.add_collection(&collection).expect("add");
+    engine.set_retain_hir(true);
+    engine.add_collection(&collection).expect("add");
+    let err = engine
+        .save_hir()
+        .expect_err("a cache missing the first four rules must not be written");
+    assert!(err.to_string().contains("4 rules"), "{err}");
+}
+
+#[test]
+fn disabling_retention_drops_retained_hir() {
+    let collection = parse_sigma_yaml(RULES).expect("parse");
+    let mut engine = Engine::new();
+    engine.set_retain_hir(true);
+    engine.add_collection(&collection).expect("add");
+    engine.set_retain_hir(false);
+    engine.set_retain_hir(true);
+    assert!(engine.save_hir().is_err());
+}
diff --git a/docs/content/library/eval.md b/docs/content/library/eval.md
index c59cadebd..f33418015 100644
--- a/docs/content/library/eval.md
+++ b/docs/content/library/eval.md
@@ -200,7 +200,8 @@ Pipeline transformations can write `rsigma.*` attributes that the engine consume
 | `Engine::with_logsource_extractor(extractor)` | Create an empty engine that prunes with a custom `ExtractLogSource` implementation. {{ added "unreleased" }} |
 | `Engine::evaluate_pruned(&event, &LogSource)` | Evaluate with a caller-resolved event logsource for conflict-based pruning, bypassing the engine's own extractor. Used by `SchemaRouter` to feed a per-event logsource resolved from explicit fields plus the recognized schema's implied logsource. |
 | `Engine::evaluate_batch(&[events])` (with `parallel`) | Batch evaluation. With the `parallel` feature, rayon parallelizes across events internally. |
-| `Engine::save_hir()` / `load_hir(&[u8])` | Serialize the engine's lowered rules to a versioned HIR cache blob and rebuild from one, a restart cache that skips parse, pipeline, and lowering. Captures rules added via the parsed-rule paths in post-pipeline, pre-filter form; re-apply filters after `load_hir`. Backed by [`rsigma-ir`](ir.md)'s `cache`. |
+| `Engine::set_retain_hir(bool)` | Keep each loaded rule's HIR so `save_hir` can write it. Off by default, since nothing on the evaluation path reads it; enable it before loading rules. {{ added "unreleased" }} |
+| `Engine::save_hir()` / `load_hir(&[u8])` | Serialize the engine's lowered rules to a versioned HIR cache blob and rebuild from one, a restart cache that skips parse, pipeline, and lowering. Captures rules added via the parsed-rule paths in post-pipeline, pre-filter form; re-apply filters after `load_hir`. `save_hir` returns `EvalError::HirCache` unless `set_retain_hir(true)` was on for every such rule. Backed by [`rsigma-ir`](ir.md)'s `cache`. |
 
 Schema classification and routing live alongside the engine: `SchemaClassifier::classify` and `classify_with_ambiguity` recognize an event's schema from declarative `SchemaSignature` predicates, `explain` reports why, and `validate_schema_config` statically checks a config. `SchemaRouter` builds one engine per pipeline-set, routes each event to its schema's engine (deriving the event's logsource from the schema for pruning), feeds one shared correlation store, and reports a per-schema pruning summary via `schema_pruning_summary`. See the [Schema Signatures reference](../reference/schema-signatures.md) and the [Schema Routing](../guide/schema-routing.md) guide.
 
diff --git a/docs/content/library/ir.md b/docs/content/library/ir.md
index f6bb564dc..7116ade52 100644
--- a/docs/content/library/ir.md
+++ b/docs/content/library/ir.md
@@ -47,7 +47,7 @@ The crate is sync-only (no tokio/reqwest).
 
 ## HIR cache
 
-`cache::*` serializes lowered rules to a versioned, self-describing blob, the on-disk HIR cache. It is what [`rsigma-eval`](eval.md)'s `Engine::save_hir` / `load_hir` use for a daemon restart cache that skips parse, pipeline, and lowering.
+`cache::*` serializes lowered rules to a versioned, self-describing blob, the on-disk HIR cache. It is what [`rsigma-eval`](eval.md)'s `Engine::save_hir` / `load_hir` use for a restart cache that skips parse, pipeline, and lowering; the engine keeps the HIR to save only after `Engine::set_retain_hir(true)`.
 
 - The blob is a `HirCacheHeader` (schema version + producing crate version) followed by the rules. `decode_rules` reads and version-checks the header before decoding the rules, rejecting a blob written under an incompatible `HIR_SCHEMA_VERSION`.
 - CBOR is the binary format: the HIR embeds `LogSource`, whose `#[serde(flatten)]` custom-key map serializes with an unknown length that fixed-layout encoders reject.
diff --git a/docs/content/reference/architecture.md b/docs/content/reference/architecture.md
index 952fc9706..149a00239 100644
--- a/docs/content/reference/architecture.md
+++ b/docs/content/reference/architecture.md
@@ -120,7 +120,7 @@ Both evaluation and conversion route through a shared intermediate representatio
 
 ### HIR to compiled rules (`rsigma-eval::compiler`)
 
-`Engine::add_collection` first runs every loaded pipeline against each rule, in priority order, then lowers each rewritten rule to HIR (`lower_rule`) and materializes it into a `CompiledRule` via `compile_to_compiled`. Materialization builds the matcher tree, a tree of nodes per detection item with the concrete `Regex` / Aho-Corasick / `IpNet` matchers, and the matcher optimizer transforms subtrees in-place. `Engine::save_hir` / `load_hir` persist and reload that lowered rule set through the HIR cache, so a warm process skips parse, pipeline, and lowering on restart.
+`Engine::add_collection` first runs every loaded pipeline against each rule, in priority order, then lowers each rewritten rule to HIR (`lower_rule`) and materializes it into a `CompiledRule` via `compile_to_compiled`. Materialization builds the matcher tree, a tree of nodes per detection item with the concrete `Regex` / Aho-Corasick / `IpNet` matchers, and the matcher optimizer transforms subtrees in-place. `Engine::save_hir` / `load_hir` persist and reload that lowered rule set through the HIR cache, so a warm process skips parse, pipeline, and lowering on restart. The engine keeps the lowered rules for this only when `Engine::set_retain_hir(true)` is set before loading; by default it drops them after compilation.
 
 The matcher optimizer makes three transparent rewrites:
 

From 4a69e544305e54e5e3aa67b69b856e083f5ff1c4 Mon Sep 17 00:00:00 2001
From: Mostafa Moradian <mostafa@tigerdata.com>
Date: Sun, 18 Oct 2026 21:00:40 +0200
Subject: [PATCH 2/6] refactor(eval): merge the bloom and no-bloom evaluation
 loops

---
 crates/rsigma-eval/src/engine/mod.rs | 133 ++++++---------------------
 1 file changed, 28 insertions(+), 105 deletions(-)

diff --git a/crates/rsigma-eval/src/engine/mod.rs b/crates/rsigma-eval/src/engine/mod.rs
index b3b00bbde..98bf3ed95 100644
--- a/crates/rsigma-eval/src/engine/mod.rs
+++ b/crates/rsigma-eval/src/engine/mod.rs
@@ -32,7 +32,7 @@ use crate::pipeline::{Pipeline, TransformedRule, apply_pipelines, transform_coll
 use crate::result::{EvaluationResult, MatchDetailLevel};
 use crate::rule_metadata::{RuleBundleMetadata, RuleMetadataLookup};
 
-use bloom_index::{BloomCache, FieldBloomIndex};
+use bloom_index::{BloomCache, BloomLookup, FieldBloomIndex};
 
 pub use filters::apply_filters;
 use filters::{
@@ -801,9 +801,12 @@ impl<L> Engine<L> {
         event_logsource: Option<&LogSource>,
     ) -> Vec<IdentifiedEvaluation> {
         if self.bloom_prefilter {
-            self.evaluate_with_bloom_path(event, event_logsource)
+            let bloom = BloomCache::new(&self.bloom_index, event);
+            self.evaluate_candidates(event, event_logsource, &bloom)
         } else {
-            self.evaluate_no_bloom_path(event, event_logsource)
+            // `NoBloom` is zero-sized, so this instantiation compiles to the
+            // same straight-line code as an engine without a bloom filter.
+            self.evaluate_candidates(event, event_logsource, &bloom_index::NoBloom)
         }
     }
 
@@ -871,14 +874,12 @@ impl<L> Engine<L> {
         }
     }
 
-    fn evaluate_no_bloom_path<E: Event>(
+    fn evaluate_candidates<E: Event, B: BloomLookup>(
         &self,
         event: &E,
         event_logsource: Option<&LogSource>,
+        bloom: &B,
     ) -> Vec<IdentifiedEvaluation> {
-        // Pass the zero-sized `NoBloom` lookup so this monomorphizes to the
-        // same straight-line code as the pre-bloom engine while still
-        // threading the configured match-detail level.
         let keep = self.cross_rule_ac_keep_mask(event);
         // `event_logsource` is `None` (the default) unless pruning is enabled,
         // leaving the loop's behaviour unchanged.
@@ -898,15 +899,7 @@ impl<L> Engine<L> {
                 pruned += 1;
                 continue;
             }
-            if let Some(mut m) =
-                evaluate_rule_with_bloom(rule, event, &bloom_index::NoBloom, self.match_detail)
-            {
-                if self.include_event
-                    && let Some(d) = m.as_detection_mut()
-                    && d.event.is_none()
-                {
-                    d.event = Some(event.to_json());
-                }
+            if let Some(m) = self.evaluate_one(rule, event, bloom) {
                 results.push(IdentifiedEvaluation {
                     result: m,
                     rule_id: rule.id.clone(),
@@ -918,47 +911,21 @@ impl<L> Engine<L> {
         results
     }
 
-    fn evaluate_with_bloom_path<E: Event>(
+    /// Evaluate one rule, attaching the event when `include_event` is set.
+    fn evaluate_one<E: Event, B: BloomLookup>(
         &self,
+        rule: &CompiledRule,
         event: &E,
-        event_logsource: Option<&LogSource>,
-    ) -> Vec<IdentifiedEvaluation> {
-        let bloom = BloomCache::new(&self.bloom_index, event);
-        let keep = self.cross_rule_ac_keep_mask(event);
-        // `event_logsource` is `None` (the default) unless pruning is enabled,
-        // leaving the loop's behaviour unchanged.
-        let candidates = self.logsource_candidates(event, event_logsource);
-        let mut results = Vec::new();
-        let mut pruned = 0u64;
-        for idx in candidates {
-            if let Some(ref mask) = keep
-                && !mask[idx]
-            {
-                continue;
-            }
-            let rule = &self.rules[idx];
-            if let Some(event_ls) = event_logsource
-                && !logsource_compatible(&rule.logsource, event_ls)
-            {
-                pruned += 1;
-                continue;
-            }
-            if let Some(mut m) = evaluate_rule_with_bloom(rule, event, &bloom, self.match_detail) {
-                if self.include_event
-                    && let Some(d) = m.as_detection_mut()
-                    && d.event.is_none()
-                {
-                    d.event = Some(event.to_json());
-                }
-                results.push(IdentifiedEvaluation {
-                    result: m,
-                    rule_id: rule.id.clone(),
-                    rule_name: rule.name.clone(),
-                });
-            }
+        bloom: &B,
+    ) -> Option<EvaluationResult> {
+        let mut m = evaluate_rule_with_bloom(rule, event, bloom, self.match_detail)?;
+        if self.include_event
+            && let Some(d) = m.as_detection_mut()
+            && d.event.is_none()
+        {
+            d.event = Some(event.to_json());
         }
-        self.record_logsource_pruned(pruned);
-        results
+        Some(m)
     }
 
     /// Fold one event's residual logsource-conflict tally into the counter.
@@ -982,16 +949,18 @@ impl<L> Engine<L> {
         event_logsource: &LogSource,
     ) -> Vec<EvaluationResult> {
         if self.bloom_prefilter {
-            self.evaluate_with_logsource_with_bloom(event, event_logsource)
+            let bloom = BloomCache::new(&self.bloom_index, event);
+            self.evaluate_matching_logsource(event, event_logsource, &bloom)
         } else {
-            self.evaluate_with_logsource_no_bloom(event, event_logsource)
+            self.evaluate_matching_logsource(event, event_logsource, &bloom_index::NoBloom)
         }
     }
 
-    fn evaluate_with_logsource_no_bloom<E: Event>(
+    fn evaluate_matching_logsource<E: Event, B: BloomLookup>(
         &self,
         event: &E,
         event_logsource: &LogSource,
+        bloom: &B,
     ) -> Vec<EvaluationResult> {
         let keep = self.cross_rule_ac_keep_mask(event);
         let mut results = Vec::new();
@@ -1003,46 +972,8 @@ impl<L> Engine<L> {
             }
             let rule = &self.rules[idx];
             if logsource_matches(&rule.logsource, event_logsource)
-                && let Some(mut m) =
-                    evaluate_rule_with_bloom(rule, event, &bloom_index::NoBloom, self.match_detail)
+                && let Some(m) = self.evaluate_one(rule, event, bloom)
             {
-                if self.include_event
-                    && let Some(d) = m.as_detection_mut()
-                    && d.event.is_none()
-                {
-                    d.event = Some(event.to_json());
-                }
-                results.push(m);
-            }
-        }
-        results
-    }
-
-    fn evaluate_with_logsource_with_bloom<E: Event>(
-        &self,
-        event: &E,
-        event_logsource: &LogSource,
-    ) -> Vec<EvaluationResult> {
-        let bloom = BloomCache::new(&self.bloom_index, event);
-        let keep = self.cross_rule_ac_keep_mask(event);
-        let mut results = Vec::new();
-        for idx in self.rule_index.candidates(event) {
-            if let Some(ref mask) = keep
-                && !mask[idx]
-            {
-                continue;
-            }
-            let rule = &self.rules[idx];
-            if logsource_matches(&rule.logsource, event_logsource)
-                && let Some(mut m) =
-                    evaluate_rule_with_bloom(rule, event, &bloom, self.match_detail)
-            {
-                if self.include_event
-                    && let Some(d) = m.as_detection_mut()
-                    && d.event.is_none()
-                {
-                    d.event = Some(event.to_json());
-                }
                 results.push(m);
             }
         }
@@ -1115,15 +1046,7 @@ impl<L> Engine<L> {
             if !keep(rule) {
                 continue;
             }
-            if let Some(mut m) =
-                evaluate_rule_with_bloom(rule, event, &bloom_index::NoBloom, self.match_detail)
-            {
-                if self.include_event
-                    && let Some(d) = m.as_detection_mut()
-                    && d.event.is_none()
-                {
-                    d.event = Some(event.to_json());
-                }
+            if let Some(m) = self.evaluate_one(rule, event, &bloom_index::NoBloom) {
                 results.push(m);
             }
         }

From 1313d503856b179bf3ea2c4324f85ec9ec4cd28a Mon Sep 17 00:00:00 2001
From: Mostafa Moradian <mostafa@tigerdata.com>
Date: Sun, 18 Oct 2026 21:04:26 +0200
Subject: [PATCH 3/6] fix(eval): share one source-reference grammar and
 validate source ids

---
 crates/rsigma-eval/README.md                  |   2 +
 crates/rsigma-eval/src/pipeline/parsing.rs    |  78 +++++++-------
 crates/rsigma-eval/src/pipeline/sources.rs    |  88 +++++++++++++++
 crates/rsigma-eval/src/pipeline/tests.rs      |  42 ++++++++
 crates/rsigma-runtime/src/sources/template.rs | 102 ++++++++++--------
 docs/content/library/eval.md                  |   1 +
 docs/content/reference/dynamic-sources.md     |   2 +
 7 files changed, 232 insertions(+), 83 deletions(-)

diff --git a/crates/rsigma-eval/README.md b/crates/rsigma-eval/README.md
index 34b391d4e..ae98b3af9 100644
--- a/crates/rsigma-eval/README.md
+++ b/crates/rsigma-eval/README.md
@@ -97,6 +97,8 @@ A non-short-circuiting, bloom-free recording evaluator that backs `rsigma engine
 | `parse_sources_file(path: &Path)` | Parse a standalone source YAML file (top-level `sources:` list) |
 | `parse_sources_dir(dir: &Path)` | Parse all `*.yml`/`*.yaml` source files in a directory |
 | `validate_source_refs(pipeline, external_ids)` | Validate that `${source.*}` references resolve against declared and/or external source IDs |
+| `sources::scan_source_templates(text)` | Find every `${source.<id>}` or `${source.<id>.<path>}` template in a string, with its id, path, and byte offset; the one reference grammar used by pipeline parsing and runtime expansion |
+| `sources::is_valid_source_id(id)` | Whether an id is declarable: ASCII letters, digits, `_`, and `-` |
 | `apply_pipelines(pipelines, rule)` | Apply all pipelines to a rule, in slice order |
 | `apply_pipelines_with_state(pipelines, rule)` | Same ordering, and returns the merged `PipelineState` (for backends) |
 | `merge_pipelines(pipelines)` | Sort a pipeline slice by `priority` in place, lower first. Despite the name it does not combine or apply them; call it before the `apply_*` / `transform_*` functions, which walk the slice as given |
diff --git a/crates/rsigma-eval/src/pipeline/parsing.rs b/crates/rsigma-eval/src/pipeline/parsing.rs
index f438f3506..d7b7c0385 100644
--- a/crates/rsigma-eval/src/pipeline/parsing.rs
+++ b/crates/rsigma-eval/src/pipeline/parsing.rs
@@ -15,7 +15,7 @@ use super::conditions::{
 use super::finalizers::Finalizer;
 use super::sources::{
     DataFormat, DynamicSource, ErrorPolicy, ExtractExpr, RefLocation, RefreshPolicy, SourceRef,
-    SourceType,
+    SourceType, is_valid_source_id, scan_source_templates,
 };
 use super::transformations::Transformation;
 use super::{Pipeline, TransformationItem};
@@ -1299,6 +1299,11 @@ pub fn parse_dynamic_source(value: &yaml_serde::Value) -> Result<DynamicSource>
         .and_then(|v| v.as_str())
         .ok_or_else(|| EvalError::Pipeline("source must have an 'id' field".to_string()))?
         .to_string();
+    if !is_valid_source_id(&id) {
+        return Err(EvalError::Pipeline(format!(
+            "source id '{id}' is invalid: use only ASCII letters, digits, '_', and '-' so ${{source.<id>}} can reference it"
+        )));
+    }
 
     let type_str = obj
         .get(ykey("type"))
@@ -1647,14 +1652,16 @@ pub fn validate_source_refs(
 // Template reference scanning
 // =============================================================================
 
-/// Regex matching `${source.<id>}` or `${source.<id>.<sub_path>}` templates.
-fn source_ref_regex() -> &'static Regex {
-    use std::sync::OnceLock;
-    static RE: OnceLock<Regex> = OnceLock::new();
-    RE.get_or_init(|| {
-        Regex::new(r"\$\{source\.([a-zA-Z_][a-zA-Z0-9_]*)(?:\.([a-zA-Z0-9_.]+))?\}")
-            .expect("source ref regex is valid")
-    })
+/// Record every `${source.*}` template in `text`, each at `location`.
+fn push_source_refs(text: &str, location: impl Fn() -> RefLocation, refs: &mut Vec<SourceRef>) {
+    for template in scan_source_templates(text) {
+        refs.push(SourceRef {
+            source_id: template.source_id.to_string(),
+            sub_path: template.sub_path.map(str::to_string),
+            location: location(),
+            raw_template: template.raw.to_string(),
+        });
+    }
 }
 
 /// Scan the entire pipeline YAML for `${source.*}` template references and
@@ -1680,16 +1687,13 @@ pub(crate) fn scan_source_refs(obj: &yaml_serde::Mapping) -> Vec<SourceRef> {
                 if let Some(include_val) = mapping.get(ykey("include"))
                     && let Some(s) = yaml_value_as_str(include_val)
                 {
-                    for cap in source_ref_regex().captures_iter(s) {
-                        refs.push(SourceRef {
-                            source_id: cap[1].to_string(),
-                            sub_path: cap.get(2).map(|m| m.as_str().to_string()),
-                            location: RefLocation::Include {
-                                transform_index: idx,
-                            },
-                            raw_template: cap[0].to_string(),
-                        });
-                    }
+                    push_source_refs(
+                        s,
+                        || RefLocation::Include {
+                            transform_index: idx,
+                        },
+                        &mut refs,
+                    );
                 }
 
                 // Scan all other string values in the mapping
@@ -1725,16 +1729,13 @@ pub(crate) fn scan_source_refs(obj: &yaml_serde::Mapping) -> Vec<SourceRef> {
 fn scan_var_value_for_refs(value: &yaml_serde::Value, var_name: &str, refs: &mut Vec<SourceRef>) {
     match value {
         yaml_serde::Value::String(s) => {
-            for cap in source_ref_regex().captures_iter(s) {
-                refs.push(SourceRef {
-                    source_id: cap[1].to_string(),
-                    sub_path: cap.get(2).map(|m| m.as_str().to_string()),
-                    location: RefLocation::Var {
-                        var_name: var_name.to_string(),
-                    },
-                    raw_template: cap[0].to_string(),
-                });
-            }
+            push_source_refs(
+                s,
+                || RefLocation::Var {
+                    var_name: var_name.to_string(),
+                },
+                refs,
+            );
         }
         yaml_serde::Value::Sequence(seq) => {
             for item in seq {
@@ -1759,17 +1760,14 @@ fn scan_yaml_value_for_refs(
 ) {
     match value {
         yaml_serde::Value::String(s) => {
-            for cap in source_ref_regex().captures_iter(s) {
-                refs.push(SourceRef {
-                    source_id: cap[1].to_string(),
-                    sub_path: cap.get(2).map(|m| m.as_str().to_string()),
-                    location: RefLocation::TransformationField {
-                        transform_index,
-                        field_name: field_name.to_string(),
-                    },
-                    raw_template: cap[0].to_string(),
-                });
-            }
+            push_source_refs(
+                s,
+                || RefLocation::TransformationField {
+                    transform_index,
+                    field_name: field_name.to_string(),
+                },
+                refs,
+            );
         }
         yaml_serde::Value::Mapping(m) => {
             for (k, v) in m {
diff --git a/crates/rsigma-eval/src/pipeline/sources.rs b/crates/rsigma-eval/src/pipeline/sources.rs
index 7c6701933..f85a2955c 100644
--- a/crates/rsigma-eval/src/pipeline/sources.rs
+++ b/crates/rsigma-eval/src/pipeline/sources.rs
@@ -6,8 +6,11 @@
 
 use std::collections::HashMap;
 use std::path::PathBuf;
+use std::sync::LazyLock;
 use std::time::Duration;
 
+use regex::Regex;
+
 // =============================================================================
 // Dynamic source declaration
 // =============================================================================
@@ -142,6 +145,50 @@ pub struct SourceRef {
     pub raw_template: String,
 }
 
+/// Whether `id` can be declared as a dynamic source id: one or more ASCII
+/// letters, digits, `_`, or `-`. These are exactly the ids a
+/// `${source.<id>}` template can name.
+pub fn is_valid_source_id(id: &str) -> bool {
+    !id.is_empty()
+        && id
+            .bytes()
+            .all(|b| b.is_ascii_alphanumeric() || b == b'_' || b == b'-')
+}
+
+/// One `${source.<id>}` or `${source.<id>.<sub_path>}` template found by
+/// [`scan_source_templates`].
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct SourceTemplate<'a> {
+    /// The whole template, from `${` to `}`.
+    pub raw: &'a str,
+    /// Byte offset of `raw` in the scanned text.
+    pub start: usize,
+    /// The referenced source id.
+    pub source_id: &'a str,
+    /// The dot-path into the source data, if any.
+    pub sub_path: Option<&'a str>,
+}
+
+/// Find every `${source.*}` template in `text`, left to right.
+///
+/// This is the single grammar for source references: pipeline parsing uses it
+/// to record [`SourceRef`]s and the runtime uses it to expand them.
+pub fn scan_source_templates(text: &str) -> impl Iterator<Item = SourceTemplate<'_>> {
+    static RE: LazyLock<Regex> = LazyLock::new(|| {
+        Regex::new(r"\$\{source\.([A-Za-z0-9_-]+)(?:\.([A-Za-z0-9_.]+))?\}")
+            .expect("source template regex is valid")
+    });
+    RE.captures_iter(text).map(|caps| {
+        let whole = caps.get(0).expect("group 0 always participates");
+        SourceTemplate {
+            raw: whole.as_str(),
+            start: whole.start(),
+            source_id: caps.get(1).expect("the id group is required").as_str(),
+            sub_path: caps.get(2).map(|m| m.as_str()),
+        }
+    })
+}
+
 /// Where in the pipeline a source reference appears.
 #[derive(Debug, Clone, PartialEq)]
 pub enum RefLocation {
@@ -170,3 +217,44 @@ pub enum SourceStatus {
     /// Source resolution failed.
     Failed,
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn valid_source_ids() {
+        for id in ["ip_blocklist", "threat-intel", "1st_feed", "A9", "_", "-"] {
+            assert!(is_valid_source_id(id), "{id}");
+        }
+        for id in ["", "bad.id", "has space", "café", "a/b", "${x}"] {
+            assert!(!is_valid_source_id(id), "{id}");
+        }
+    }
+
+    #[test]
+    fn scan_finds_templates_with_offsets() {
+        let text = "a ${source.threat-intel.ips.0} b ${source.1st_feed} ${source.bad id}";
+        let found: Vec<_> = scan_source_templates(text).collect();
+        assert_eq!(
+            found,
+            [
+                SourceTemplate {
+                    raw: "${source.threat-intel.ips.0}",
+                    start: 2,
+                    source_id: "threat-intel",
+                    sub_path: Some("ips.0"),
+                },
+                SourceTemplate {
+                    raw: "${source.1st_feed}",
+                    start: 33,
+                    source_id: "1st_feed",
+                    sub_path: None,
+                },
+            ]
+        );
+        for t in &found {
+            assert_eq!(&text[t.start..t.start + t.raw.len()], t.raw);
+        }
+    }
+}
diff --git a/crates/rsigma-eval/src/pipeline/tests.rs b/crates/rsigma-eval/src/pipeline/tests.rs
index 36ad4efd4..202d6f0a0 100644
--- a/crates/rsigma-eval/src/pipeline/tests.rs
+++ b/crates/rsigma-eval/src/pipeline/tests.rs
@@ -1456,6 +1456,48 @@ transformations:
     assert_eq!(pipeline.source_refs[0].source_id, "some_external_source");
 }
 
+#[test]
+fn test_digit_leading_and_hyphenated_source_refs_are_dynamic() {
+    // Ids the runtime expands must also register as references here, or a
+    // pipeline whose only reference uses one is treated as static and keeps
+    // the literal template.
+    let yaml = r#"
+name: Id Shapes
+vars:
+  feed: "${source.1st_feed}"
+  intel: "${source.threat-intel.ips}"
+transformations:
+  - type: value_placeholders
+"#;
+    let pipeline = parse_pipeline(yaml).unwrap();
+    assert!(pipeline.is_dynamic());
+    let ids: Vec<_> = pipeline
+        .source_refs
+        .iter()
+        .map(|r| (r.source_id.as_str(), r.sub_path.as_deref()))
+        .collect();
+    assert_eq!(ids, [("1st_feed", None), ("threat-intel", Some("ips"))]);
+}
+
+#[test]
+fn test_source_ids_templates_cannot_reference_are_rejected() {
+    for id in ["bad.id", "has space", "\"\"", "café"] {
+        let yaml = format!(
+            "sources:\n  - id: {id}\n    type: file\n    path: /tmp/x.json\n    format: json\n"
+        );
+        let err = parse_source_list(&yaml)
+            .expect_err(&format!("id {id} must be rejected"))
+            .to_string();
+        assert!(err.contains("is invalid"), "id {id}: {err}");
+    }
+    for id in ["threat-intel", "1st_feed", "ip_blocklist"] {
+        let yaml = format!(
+            "sources:\n  - id: {id}\n    type: file\n    path: /tmp/x.json\n    format: json\n"
+        );
+        assert_eq!(parse_source_list(&yaml).unwrap()[0].id, id);
+    }
+}
+
 #[test]
 fn test_unknown_source_type_fails() {
     let yaml = r#"
diff --git a/crates/rsigma-runtime/src/sources/template.rs b/crates/rsigma-runtime/src/sources/template.rs
index 5ef9a31c5..f67dff97b 100644
--- a/crates/rsigma-runtime/src/sources/template.rs
+++ b/crates/rsigma-runtime/src/sources/template.rs
@@ -5,12 +5,8 @@
 
 use std::collections::HashMap;
 
-use regex::Regex;
 use rsigma_eval::Pipeline;
-use std::sync::LazyLock;
-
-static SOURCE_TEMPLATE_RE: LazyLock<Regex> =
-    LazyLock::new(|| Regex::new(r"\$\{source\.([a-zA-Z0-9_]+)(?:\.([a-zA-Z0-9_.]+))?\}").unwrap());
+use rsigma_eval::pipeline::sources::{SourceTemplate, scan_source_templates};
 
 /// Expands `${source.*}` template references in a pipeline using resolved source data.
 pub struct TemplateExpander;
@@ -54,53 +50,41 @@ impl TemplateExpander {
         }
 
         // If the entire value is a single template reference, replace it directly
-        if let Some(caps) = SOURCE_TEMPLATE_RE.captures(value)
-            && caps.get(0).unwrap().as_str() == value
+        if let Some(template) = scan_source_templates(value).next()
+            && template.raw == value
         {
-            let source_id = caps.get(1).unwrap().as_str();
-            let sub_path = caps.get(2).map(|m| m.as_str());
-
-            if let Some(data) = resolved.get(source_id) {
-                let target = if let Some(path) = sub_path {
-                    navigate_path(data, path)
-                } else {
-                    Some(data)
-                };
-
-                if let Some(val) = target {
-                    return Some(json_to_string_vec(val));
-                }
-            }
-
-            return None;
+            return resolve(&template, resolved).map(json_to_string_vec);
         }
 
         // Otherwise, do substring replacement (inline templates within larger strings)
-        let result = SOURCE_TEMPLATE_RE
-            .replace_all(value, |caps: &regex::Captures| {
-                let source_id = caps.get(1).unwrap().as_str();
-                let sub_path = caps.get(2).map(|m| m.as_str());
-
-                if let Some(data) = resolved.get(source_id) {
-                    let target = if let Some(path) = sub_path {
-                        navigate_path(data, path)
-                    } else {
-                        Some(data)
-                    };
-
-                    if let Some(val) = target {
-                        return json_to_single_string(val);
-                    }
-                }
-
-                caps.get(0).unwrap().as_str().to_string()
-            })
-            .to_string();
+        let mut result = String::with_capacity(value.len());
+        let mut copied = 0;
+        for template in scan_source_templates(value) {
+            result.push_str(&value[copied..template.start]);
+            match resolve(&template, resolved) {
+                Some(val) => result.push_str(&json_to_single_string(val)),
+                None => result.push_str(template.raw),
+            }
+            copied = template.start + template.raw.len();
+        }
+        result.push_str(&value[copied..]);
 
         Some(vec![result])
     }
 }
 
+/// The resolved data a template points at, if its source and path exist.
+fn resolve<'a>(
+    template: &SourceTemplate<'_>,
+    resolved: &'a HashMap<String, serde_json::Value>,
+) -> Option<&'a serde_json::Value> {
+    let data = resolved.get(template.source_id)?;
+    match template.sub_path {
+        Some(path) => navigate_path(data, path),
+        None => Some(data),
+    }
+}
+
 /// Navigate a dot-separated path into a JSON value.
 ///
 /// E.g., `"field_mapping.sysmon"` navigates `data["field_mapping"]["sysmon"]`.
@@ -179,6 +163,38 @@ mod tests {
         );
     }
 
+    #[test]
+    fn expand_digit_leading_and_hyphenated_ids() {
+        let mut vars = HashMap::new();
+        vars.insert("feed".to_string(), vec!["${source.1st_feed}".to_string()]);
+        vars.insert(
+            "url".to_string(),
+            vec!["https://${source.threat-intel.host}/x?id=${source.missing}".to_string()],
+        );
+        let pipeline = Pipeline {
+            name: "test".to_string(),
+            priority: 0,
+            vars,
+            transformations: vec![],
+            finalizers: vec![],
+            source_refs: vec![],
+        };
+        let resolved = HashMap::from([
+            ("1st_feed".to_string(), serde_json::json!(["a", "b"])),
+            (
+                "threat-intel".to_string(),
+                serde_json::json!({"host": "intel.example"}),
+            ),
+        ]);
+
+        let expanded = TemplateExpander::expand(&pipeline, &resolved);
+        assert_eq!(expanded.vars["feed"], ["a", "b"]);
+        assert_eq!(
+            expanded.vars["url"],
+            ["https://intel.example/x?id=${source.missing}"]
+        );
+    }
+
     #[test]
     fn expand_nested_path() {
         let mut vars = HashMap::new();
diff --git a/docs/content/library/eval.md b/docs/content/library/eval.md
index f33418015..22034e79e 100644
--- a/docs/content/library/eval.md
+++ b/docs/content/library/eval.md
@@ -38,6 +38,7 @@ serde_json = "1"   # only if you use the JsonEvent shim
 | `Pipeline` | Parsed processing pipeline. Applied to rules at `add_collection` time, in priority order. |
 | `ConditionSet<T>` | One pipeline condition scope: identifier-keyed conditions, `and`/`or` linking, optional negation, and an optional expression. `TransformationItem` has one set each for rule, detection-item, and field-name conditions. {{ added "0.24.0" }} |
 | `pipeline::parse_pipeline(&str) -> Result<Pipeline>` | Parse a pipeline YAML string. |
+| `pipeline::sources::scan_source_templates(&str)` | Find every `${source.<id>}` and `${source.<id>.<path>}` template in a string, with its id, path, and byte offset. It is the single reference grammar shared by pipeline parsing and `rsigma-runtime`'s expansion. `is_valid_source_id` checks the id character set (ASCII letters, digits, `_`, `-`) that source declarations must use. {{ added "unreleased" }} |
 | `TransformedRule` + `transform_rule` / `transform_collection` | Apply pipelines and hand back the rewritten rule, the transformation ids that fired, and the merged `PipelineState`, without compiling or loading. `Engine::transform_rule` / `Engine::transform_collection` do the same over an engine's configured pipelines. |
 | `apply_filters(&SigmaCollection) -> Vec<SigmaRule>` | Merge each filter into the detection rules it targets, as pySigma does when a collection loads, and return the rules for pipelines to transform. Filters target rules the same way `Engine::apply_filter` does. Conversion uses it. {{ added "0.24.0" }} |
 | `LogSourceExtractor` | Derives an event's `LogSource` from configurable fields plus optional static defaults, for conflict-based logsource pruning. Pass to `Engine::set_logsource_extractor`. |
diff --git a/docs/content/reference/dynamic-sources.md b/docs/content/reference/dynamic-sources.md
index 3f98c404b..bb0c98823 100644
--- a/docs/content/reference/dynamic-sources.md
+++ b/docs/content/reference/dynamic-sources.md
@@ -71,6 +71,8 @@ sources:
 
 The full Rust type lives at [`rsigma_eval::pipeline::sources::DynamicSource`](https://github.com/timescale/rsigma/blob/main/crates/rsigma-eval/src/pipeline/sources.rs). The parser is at [`rsigma_eval::pipeline::parsing`](https://github.com/timescale/rsigma/blob/main/crates/rsigma-eval/src/pipeline/parsing.rs).
 
+A source ID is one or more ASCII letters, digits, `_`, or `-` (for example `ip_blocklist`, `threat-intel`, or `1st_feed`), which is exactly what a `${source.<id>}` reference can name. A declaration with any other character, such as a dot or a space, fails to load with an error naming the ID. {{ added "unreleased" }}
+
 ### Collision semantics
 
 Source IDs must be unique across every `--source` file (and across every directory the flag expands into). If the same ID appears in two different declaration sites the daemon exits at startup with an error naming both file paths, so there is exactly one canonical declaration site per source ID.

From ba2a0a9d5dae78f79b26ab3aac5e42cb5ee9271e Mon Sep 17 00:00:00 2001
From: Mostafa Moradian <mostafa@tigerdata.com>
Date: Sun, 18 Oct 2026 21:06:27 +0200
Subject: [PATCH 4/6] refactor: remove dead items and a stale dead_code
 allowance

---
 crates/rsigma-cli/src/exit_code.rs            |  1 -
 crates/rsigma-lsp/src/diagnostics.rs          |  8 ------
 .../rsigma-runtime/src/enrichment/template.rs | 12 +-------
 crates/rstix/src/taxii/pagination.rs          | 28 -------------------
 crates/rstix/src/taxii/request.rs             |  3 --
 5 files changed, 1 insertion(+), 51 deletions(-)

diff --git a/crates/rsigma-cli/src/exit_code.rs b/crates/rsigma-cli/src/exit_code.rs
index 24f942e56..00a46b49d 100644
--- a/crates/rsigma-cli/src/exit_code.rs
+++ b/crates/rsigma-cli/src/exit_code.rs
@@ -8,7 +8,6 @@
 /// For `eval`: events were processed (detections may or may not have fired).
 /// For `lint`: no findings at the configured severity threshold.
 /// For `validate`: all rules parsed and compiled.
-#[allow(dead_code)]
 pub const SUCCESS: i32 = 0;
 
 /// Findings were produced.
diff --git a/crates/rsigma-lsp/src/diagnostics.rs b/crates/rsigma-lsp/src/diagnostics.rs
index 9bc8dd96f..9423c6a5a 100644
--- a/crates/rsigma-lsp/src/diagnostics.rs
+++ b/crates/rsigma-lsp/src/diagnostics.rs
@@ -8,14 +8,6 @@ use tower_lsp_server::ls_types::*;
 
 use crate::position::{LineIndex, resolve_path};
 
-/// Produce LSP diagnostics for a Sigma YAML document (default config).
-///
-/// Convenience wrapper around [`diagnose_with_config`] with no suppression.
-#[allow(dead_code)]
-pub fn diagnose(text: &str) -> Vec<Diagnostic> {
-    diagnose_with_config(text, &LintConfig::default())
-}
-
 /// Produce LSP diagnostics with lint config suppression.
 pub fn diagnose_with_config(text: &str, config: &LintConfig) -> Vec<Diagnostic> {
     let index = LineIndex::new(text);
diff --git a/crates/rsigma-runtime/src/enrichment/template.rs b/crates/rsigma-runtime/src/enrichment/template.rs
index 71eae966b..c7aa42260 100644
--- a/crates/rsigma-runtime/src/enrichment/template.rs
+++ b/crates/rsigma-runtime/src/enrichment/template.rs
@@ -41,9 +41,7 @@ use regex::Regex;
 use rsigma_eval::{EvaluationResult, ResultBody};
 use rsigma_parser::Level;
 
-use super::{
-    EnrichError, EnrichErrorKind, Enricher, EnricherKind, OnError, Scope, inject_enrichment,
-};
+use super::{EnrichError, Enricher, EnricherKind, OnError, Scope, inject_enrichment};
 
 /// Matches `${<contents>}` where contents is anything except `}`.
 ///
@@ -489,14 +487,6 @@ impl Enricher for TemplateEnricher {
     }
 }
 
-// `EnrichError` / `EnrichErrorKind` are referenced by the trait definition
-// above via `super::*`; this `_use` keeps unused-import warnings off when
-// future expansions fold in custom errors here without changing the bound.
-#[allow(dead_code)]
-fn _use_err(_e: EnrichError) -> EnrichErrorKind {
-    EnrichErrorKind::TemplateRender(String::new())
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;
diff --git a/crates/rstix/src/taxii/pagination.rs b/crates/rstix/src/taxii/pagination.rs
index c87dacf8e..749e52b97 100644
--- a/crates/rstix/src/taxii/pagination.rs
+++ b/crates/rstix/src/taxii/pagination.rs
@@ -6,63 +6,35 @@ use super::TaxiiError;
 use super::filter::{ObjectByIdFilter, TaxiiFilter, VersionsQueryFilter};
 
 pub(crate) trait PaginatedFilter {
-    #[allow(dead_code)]
-    fn added_after(&self) -> Option<TaxiiTimestamp>;
     fn set_added_after(&mut self, ts: Option<TaxiiTimestamp>);
-    #[allow(dead_code)]
-    fn next(&self) -> Option<&str>;
     fn set_next(&mut self, cursor: Option<String>);
 }
 
 impl PaginatedFilter for TaxiiFilter {
-    fn added_after(&self) -> Option<TaxiiTimestamp> {
-        self.added_after.clone()
-    }
-
     fn set_added_after(&mut self, ts: Option<TaxiiTimestamp>) {
         self.added_after = ts;
     }
 
-    fn next(&self) -> Option<&str> {
-        self.next.as_deref()
-    }
-
     fn set_next(&mut self, cursor: Option<String>) {
         self.next = cursor;
     }
 }
 
 impl PaginatedFilter for ObjectByIdFilter {
-    fn added_after(&self) -> Option<TaxiiTimestamp> {
-        self.added_after.clone()
-    }
-
     fn set_added_after(&mut self, ts: Option<TaxiiTimestamp>) {
         self.added_after = ts;
     }
 
-    fn next(&self) -> Option<&str> {
-        self.next.as_deref()
-    }
-
     fn set_next(&mut self, cursor: Option<String>) {
         self.next = cursor;
     }
 }
 
 impl PaginatedFilter for VersionsQueryFilter {
-    fn added_after(&self) -> Option<TaxiiTimestamp> {
-        self.added_after.clone()
-    }
-
     fn set_added_after(&mut self, ts: Option<TaxiiTimestamp>) {
         self.added_after = ts;
     }
 
-    fn next(&self) -> Option<&str> {
-        self.next.as_deref()
-    }
-
     fn set_next(&mut self, cursor: Option<String>) {
         self.next = cursor;
     }
diff --git a/crates/rstix/src/taxii/request.rs b/crates/rstix/src/taxii/request.rs
index 9febf31d1..07adab2af 100644
--- a/crates/rstix/src/taxii/request.rs
+++ b/crates/rstix/src/taxii/request.rs
@@ -38,8 +38,6 @@ pub(crate) struct TaxiiResponse {
     pub status: StatusCode,
     pub headers: HeaderMap,
     pub body: Vec<u8>,
-    #[allow(dead_code)]
-    pub clock_skew: Option<i64>,
 }
 
 impl TaxiiHttp {
@@ -170,7 +168,6 @@ impl TaxiiHttp {
             status,
             headers,
             body,
-            clock_skew,
         })
     }
 

From 8f5deeead566e4e9d1a522724168aa8abecab384 Mon Sep 17 00:00:00 2001
From: Mostafa Moradian <mostafa@tigerdata.com>
Date: Sun, 18 Oct 2026 21:17:44 +0200
Subject: [PATCH 5/6] docs(changelog): note opt-in HIR retention and the shared
 source grammar

---
 CHANGELOG.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 297f4e7e0..0027c7811 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,6 +4,12 @@ All notable changes to RSigma are documented in this file. Each entry correspond
 
 ## [Unreleased]
 
+### Engines no longer retain rule HIR by default, and source references share one grammar
+
+`Engine` kept every loaded rule's lowered HIR beside its compiled form, only so `save_hir` could write a restart cache, which nothing in the CLI, daemon, MCP server, or LSP calls. Retention is now opt-in through `Engine::set_retain_hir(true)`, which must be on before rules are loaded; loading the 3,132 SigmaHQ rules with `engine eval` peaks about 12 to 16 MB (roughly 6%) lower without it. This changes library behavior: `save_hir` now returns `EvalError::HirCache` unless retention was on for every rule loaded from parsed Sigma, rather than writing a cache that silently misses rules.
+
+The `${source.*}` reference grammar existed twice, and the copies disagreed. Pipeline parsing ignored ids starting with a digit, so a pipeline whose only reference was `${source.1st_feed}` was treated as static and kept the literal template, and ids containing `-`, such as `threat-intel`, matched neither copy. Both sides now use `rsigma_eval::pipeline::sources::scan_source_templates`, and a reference id is one or more ASCII letters, digits, `_`, or `-`. A source declaration whose id uses any other character, such as a dot or a space, now fails to load with an error naming the id, since no template could reference it.
+
 ### Percentile and median checks no longer sort the window (#619)
 
 `value_percentile` and `value_median` correlations sorted every finite value in the group's window on each event fed into the group. They now select the needed ranks with `select_nth_unstable_by`, which is O(n) on average and returns exactly the same values, including interpolated percentiles and even-count medians. In the new `correlation_numeric_window` benchmark, a group checked against a 5,000-value window processes events about 4x faster. Chained correlations also stop cloning every fired result, including its retained events in full event mode, just to read which correlation fired and for which group.

From 76efebda5e23992006417de0b3868cdd7b74f811 Mon Sep 17 00:00:00 2001
From: Mostafa Moradian <mostafa@tigerdata.com>
Date: Sun, 18 Oct 2026 21:18:08 +0200
Subject: [PATCH 6/6] docs(changelog): add the PR number to the HIR retention
 entry

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0027c7811..478d447b6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,7 +4,7 @@ All notable changes to RSigma are documented in this file. Each entry correspond
 
 ## [Unreleased]
 
-### Engines no longer retain rule HIR by default, and source references share one grammar
+### Engines no longer retain rule HIR by default, and source references share one grammar (#620)
 
 `Engine` kept every loaded rule's lowered HIR beside its compiled form, only so `save_hir` could write a restart cache, which nothing in the CLI, daemon, MCP server, or LSP calls. Retention is now opt-in through `Engine::set_retain_hir(true)`, which must be on before rules are loaded; loading the 3,132 SigmaHQ rules with `engine eval` peaks about 12 to 16 MB (roughly 6%) lower without it. This changes library behavior: `save_hir` now returns `EvalError::HirCache` unless retention was on for every rule loaded from parsed Sigma, rather than writing a cache that silently misses rules.